#!/bin/bash

# This script scrapes digitaldante.columbia.edu to get the Petrocchi edition of the Divine Comedy.
# It downloads the source HTML from each canto's digitaldante webpage
# Then it isolates the HTML canto portion of the webpage
# Then it removes the HTML on the left and right of each line
# Then it removes the remaining bits of HTML (HTML to ASCII, removing <em></em> tags)
# The final canto is in a final named canticle-cantonumber

I="inferno"
R="purgatorio"
P="paradiso"

canticles=( "$I" "$R" "$P" )

rm -rf "./inferno/" && mkdir "inferno"

rm -rf "./purgatorio/" && mkdir "purgatorio"

rm -rf "./paradiso/" && mkdir "paradiso"

BASE_URL="https://digitaldante.columbia.edu/dante/divine-comedy/"


### SOURCE CODE DOWNLOAD
for canticle in "${canticles[@]}";
do
	BASE_DIR="$canticle/"
	for i in {1..33};
	do
		FILE_CANTO_SOURCE="$BASE_DIR$canticle-$i.source"	# Source HTML from digitaldante
		FILE_CANTO_HTML="$BASE_DIR$canticle-$i.html"		# HTML of only canto, other source HTML removed
		FILE_CANTO_BITS="$BASE_DIR$canticle-$i.bits"		# Bits (tags) remain, surrounding HTML removed
		FILE_CANTO="$BASE_DIR$canticle-$i"			# Final canto

		echo "Processing $canticle-$i"

		CANTO_URL="$BASE_URL$canticle/$canticle-$i/"
		wget --quiet -O - "$CANTO_URL" >> "$FILE_CANTO_SOURCE"

# Isolate canto in HTML
		CANTO_START=$(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | head -n1)
		CANTO_END=$(( $(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | sed -n '2p' ) - 6 ))

		sed -i "$CANTO_END,$(wc -l $FILE_CANTO_SOURCE | cut -d' ' -f1)d" "$FILE_CANTO_SOURCE" && sed -i "1,$CANTO_START d" $FILE_CANTO_SOURCE && mv "$FILE_CANTO_SOURCE" "$FILE_CANTO_HTML"

# Remove surrounding HTML
		cat "$FILE_CANTO_HTML" | awk -F '</span>|</span> ' '{print $2}' | awk -F '<br />|</p>' '{print $1}' >> tmp && mv tmp "$FILE_CANTO_BITS"
		rm "$FILE_CANTO_HTML"

# Replace weird punctuation conversions in the donwload process (?)
		WEIRD_PUNCT=( '&#8211;' '&#8212;' '&#8216;' '&#8217;' '&#8220;' '&#8221;' )
		PUNCT_REPLACE=( "—" "–—" "‘" "’" "“" "”" )
		
		for punct in ${!WEIRD_PUNCT[@]};
		do
			[[ ! -z $(grep "${WEIRD_PUNCT["$punct"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${WEIRD_PUNCT["$punct"]}"/"${PUNCT_REPLACE["$punct"]}"/g" "$FILE_CANTO_BITS"
		done

# Remove HTML tags for italics and the like
		HTML_TAGS=( '<i>' '<\/i>' '<em>' '<\/em>' )
		for tag in ${!HTML_TAGS[@]};
		do
			[[ ! -z $(grep "${HTML_TAGS["$tag"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${HTML_TAGS["$tag"]}"//g" "$FILE_CANTO_BITS"
		done
		mv "$FILE_CANTO_BITS" "$FILE_CANTO"

# Remove single non-recognized characters `�` that might occur at the beginning of each line (discovered on line 106 of Inferno 2, for example. These � mess up other scripts.)
		#if [[ $(cut -c1) == [a-zA-Z«“‘] ]]
		#mv "$FILE_CANTO_BITS" "$FILE_CANTO"
	done
done