You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
72 lines
2.8 KiB
72 lines
2.8 KiB
#!/bin/bash
|
|
|
|
# This script scrapes digitaldante.columbia.edu to get the Petrocchi edition of the Divine Comedy.
|
|
# It downloads the source HTML from each canto's digitaldante webpage
|
|
# Then it isolates the HTML canto portion of the webpage
|
|
# Then it removes the HTML on the left and right of each line
|
|
# Then it removes the remaining bits of HTML (HTML to ASCII, removing <em></em> tags)
|
|
# The final canto is in a final named canticle-cantonumber
|
|
|
|
I="inferno"
|
|
R="purgatorio"
|
|
P="paradiso"
|
|
|
|
canticles=( "$I" "$R" "$P" )
|
|
|
|
rm -rf "./inferno/" && mkdir "inferno"
|
|
|
|
rm -rf "./purgatorio/" && mkdir "purgatorio"
|
|
|
|
rm -rf "./paradiso/" && mkdir "paradiso"
|
|
|
|
BASE_URL="https://digitaldante.columbia.edu/dante/divine-comedy/"
|
|
|
|
|
|
### SOURCE CODE DOWNLOAD
|
|
for canticle in "${canticles[@]}";
|
|
do
|
|
BASE_DIR="$canticle/"
|
|
for i in {1..33};
|
|
do
|
|
FILE_CANTO_SOURCE="$BASE_DIR$canticle-$i.source" # Source HTML from digitaldante
|
|
FILE_CANTO_HTML="$BASE_DIR$canticle-$i.html" # HTML of only canto, other source HTML removed
|
|
FILE_CANTO_BITS="$BASE_DIR$canticle-$i.bits" # Bits (tags) remain, surrounding HTML removed
|
|
FILE_CANTO="$BASE_DIR$canticle-$i" # Final canto
|
|
|
|
echo "Processing $canticle-$i"
|
|
|
|
CANTO_URL="$BASE_URL$canticle/$canticle-$i/"
|
|
wget --quiet -O - "$CANTO_URL" >> "$FILE_CANTO_SOURCE"
|
|
|
|
# Isolate canto in HTML
|
|
CANTO_START=$(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | head -n1)
|
|
CANTO_END=$(( $(grep -n translation-entry "$FILE_CANTO_SOURCE" | cut -d: -f1 | sed -n '2p' ) - 6 ))
|
|
|
|
sed -i "$CANTO_END,$(wc -l $FILE_CANTO_SOURCE | cut -d' ' -f1)d" "$FILE_CANTO_SOURCE" && sed -i "1,$CANTO_START d" $FILE_CANTO_SOURCE && mv "$FILE_CANTO_SOURCE" "$FILE_CANTO_HTML"
|
|
|
|
# Remove surrounding HTML
|
|
cat "$FILE_CANTO_HTML" | awk -F '</span>|</span> ' '{print $2}' | awk -F '<br />|</p>' '{print $1}' >> tmp && mv tmp "$FILE_CANTO_BITS"
|
|
rm "$FILE_CANTO_HTML"
|
|
|
|
# Replace weird punctuation conversions in the donwload process (?)
|
|
WEIRD_PUNCT=( '–' '—' '‘' '’' '“' '”' )
|
|
PUNCT_REPLACE=( "—" "–—" "‘" "’" "“" "”" )
|
|
|
|
for punct in ${!WEIRD_PUNCT[@]};
|
|
do
|
|
[[ ! -z $(grep "${WEIRD_PUNCT["$punct"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${WEIRD_PUNCT["$punct"]}"/"${PUNCT_REPLACE["$punct"]}"/g" "$FILE_CANTO_BITS"
|
|
done
|
|
|
|
# Remove HTML tags for italics and the like
|
|
HTML_TAGS=( '<i>' '<\/i>' '<em>' '<\/em>' )
|
|
for tag in ${!HTML_TAGS[@]};
|
|
do
|
|
[[ ! -z $(grep "${HTML_TAGS["$tag"]}" "$FILE_CANTO_BITS") ]] && sed -i "s/"${HTML_TAGS["$tag"]}"//g" "$FILE_CANTO_BITS"
|
|
done
|
|
mv "$FILE_CANTO_BITS" "$FILE_CANTO"
|
|
|
|
# Remove single non-recognized characters `�` that might occur at the beginning of each line (discovered on line 106 of Inferno 2, for example. These � mess up other scripts.)
|
|
#if [[ $(cut -c1) == [a-zA-Z«“‘] ]]
|
|
#mv "$FILE_CANTO_BITS" "$FILE_CANTO"
|
|
done
|
|
done
|
|
|