#!/bin/bash

# This script scrapes digitaldante.columbia.edu to get the Petrocchi edition of the Divine Comedy.
# It downloads the source HTML from each canto's digitaldante webpage
# Then it isolates the HTML canto portion of the webpage
# Then it removes the HTML on the left and right of each line
# Then it removes the remaining bits of HTML (HTML to ASCII, removing <em></em> tags)
# The final canto is in a final named canticle-cantonumber

#I="inferno"
#R="purgatorio"
#P="paradiso"

#canticles=( "$I" "$R" "$P" )

BASE_URL="https://digitaldante.columbia.edu/dante/divine-comedy/"


### SOURCE CODE DOWNLOAD
#for canticle in "${canticles[@]}";
#do
canticle="inferno"
	BASE_DIR="$canticle/"
	i="34"
	FILE_CANTO_SOURCE="$BASE_DIR$canticle-$i.source"	# Source HTML from digitaldante
	FILE_CANTO_HTML="$BASE_DIR$canticle-$i.html"		# HTML of only canto, other source HTML removed
	FILE_CANTO_BITS="$BASE_DIR$canticle-$i.bits"		# Bits (tags) remain, surrounding HTML removed
	FILE_CANTO="$BASE_DIR$canticle-$i"			# Final canto

	echo "Processing $canticle-$i"

	CANTO_URL="$BASE_URL$canticle/$canticle-$i/"
	wget --quiet -O - "$CANTO_URL" >> "$FILE_CANTO_SOURCE"

# Isolate canto in HTML
	CANTO_START=$(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | head -n1)
	CANTO_END=$(( $(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | sed -n '2p' ) - 6 ))

	sed -i "$CANTO_END,$(wc -l $FILE_CANTO_SOURCE | cut -d' ' -f1)d" "$FILE_CANTO_SOURCE" && sed -i "1,$CANTO_START d" $FILE_CANTO_SOURCE && mv "$FILE_CANTO_SOURCE" "$FILE_CANTO_HTML"

# Remove surrounding HTML
	cat "$FILE_CANTO_HTML" | awk -F '</span>|</span> ' '{print $2}' | awk -F '<br />|</p>' '{print $1}' >> tmp && mv tmp "$FILE_CANTO_BITS"

# Replace weird punctuation conversions in the donwload process (?)
	WEIRD_PUNCT=( '&#8211;' '&#8212;' '&#8216;' '&#8217;' '&#8220;' '&#8221;' )
	PUNCT_REPLACE=( "—" "–—" "‘" "’" "“" "”" )
		
		
	for punct in ${!WEIRD_PUNCT[@]};
	do
		[[ ! -z $(grep "${WEIRD_PUNCT["$punct"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${WEIRD_PUNCT["$punct"]}"/"${PUNCT_REPLACE["$punct"]}"/g" "$FILE_CANTO_BITS"
	done

# Remove HTML tags for italics and the like
	HTML_TAGS=( '<i>' '<\/i>' '<em>' '<\/em>' )
	for tag in ${!HTML_TAGS[@]};
	do
		[[ ! -z $(grep "${HTML_TAGS["$tag"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${HTML_TAGS["$tag"]}"//g" "$FILE_CANTO_BITS"
	done
	mv "$FILE_CANTO_BITS" "$FILE_CANTO"
	rm "$BASE_DIR$canticle-$i.html"
#done