You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
171 lines
7.4 KiB
171 lines
7.4 KiB
import re
|
|
import os
|
|
import xml.etree.ElementTree as ET
|
|
import glob
|
|
import shutil
|
|
|
|
"""
|
|
This script takes xml canto files and counts the number of unique words as
|
|
defined by the value of `<w>` nodes.
|
|
|
|
8 files are generated: csv and html of cantos
|
|
- sorted numerically and alphabetically
|
|
- sorted numerically and alphabetically with blacklist applied
|
|
|
|
Run this once in each canticle dir under comedy-xml.d/
|
|
|
|
You need to remove the csv and html files currently under comedy-wordcount.d/.
|
|
"""
|
|
|
|
cantoDir = os.path.abspath(os.getcwd())
|
|
|
|
blacklist = ('e', 'che', 'la', 'di', 'a', 'per', '’l', 'mi', 'tu', 'con',
|
|
'lo', 'sì', 'sua', 'li', 'del', 'si', 'nel', 'era', 'ch’i’', 'è',
|
|
'là', 'le', 'de', 'quella', 'poi', 'ma', 'il', 'già', '’n', 'una',
|
|
'tanto', 'se’', 'quel', 'qual', 'perché', 'in', 'come', 'al', 'da',
|
|
'un', 'i', 'ti', 'a', 'per', 'con', 'lo', 'al', 'poi', 'E', 'il')
|
|
|
|
def createWordCountDict(cantoFile):
|
|
"""
|
|
for node in xpathQueryAllWordNodes:
|
|
if xpathValue(node) in wordCountDict(keys):
|
|
wordCountDict[node] += 1
|
|
else:
|
|
wordCountDict[xpathValue(node)] = 1
|
|
"""
|
|
wordCountDict = dict()
|
|
tree = ET.parse(cantoFile)
|
|
root = tree.getroot()
|
|
words = root.findall(".//{http://www.tei-c.org/ns/1.0}w")
|
|
for word in words:
|
|
if word.text in wordCountDict:
|
|
wordCountDict[word.text] += 1
|
|
else:
|
|
wordCountDict[word.text] = 1
|
|
return wordCountDict
|
|
|
|
def numSort(wordCountDict):
|
|
# Need reverse = True to sort greatest to least
|
|
wordCountDictNumSorted = dict(sorted(wordCountDict.items(), key=lambda item: item[1], reverse = True))
|
|
return wordCountDictNumSorted
|
|
|
|
def alphaSort(wordCountDict):
|
|
# Capital letters come first with this implementation
|
|
wordCountDictAlphaSorted = dict(sorted(wordCountDict.items()))
|
|
return wordCountDictAlphaSorted
|
|
|
|
def blackSort(wordCountDictSorted):
|
|
sortedCopy = dict(wordCountDictSorted)
|
|
for word in wordCountDictSorted:
|
|
if word in blacklist:
|
|
del sortedCopy[word]
|
|
return sortedCopy
|
|
|
|
def dictToCsv(sortedDict, sortType, cantoFile):
|
|
"""
|
|
write a sorted dict to a CSV file for that canto
|
|
name it based on sorting type (numerical or alphabetical)
|
|
abbreviate sortType as `num` or `alpha`
|
|
"""
|
|
csvCanto = cantoFile.split('.')[0] + '-' + sortType + '.csv'
|
|
cantoNum = str(int(cantoFile.split('-')[0]))
|
|
canticle = cantoFile.split('-')[1]
|
|
|
|
with open(csvCanto, 'w') as csvCanto:
|
|
for word, count in sortedDict.items():
|
|
csvInput = word + ',' + str(count) + '\n'
|
|
csvCanto.write(csvInput)
|
|
csvCanto.close()
|
|
|
|
def dictToHtml(sortedDict, sortType, cantoFile):
|
|
"""
|
|
take a sorted dict in any permutation (i.e. num-black) and
|
|
write the word and its count into an html table
|
|
with some html boilerplate with useful information
|
|
"""
|
|
htmlCanto = cantoFile.split('.')[0] + '-' + sortType + '.html'
|
|
with open(htmlCanto, 'w') as htmlCanto: # open in binary mode to write encoded html entities
|
|
cantoNum = str(int(cantoFile.split('-')[0]))
|
|
canticle = cantoFile.split('-')[1].title()
|
|
canticleAbbrv = canticle[0:3]
|
|
if sortType == 'num':
|
|
sortInformation = 'Words sorted by most to least occurrences.'
|
|
elif sortType == 'num-black':
|
|
sortInformation = 'Words sorted by most to least occurrences, with blacklist applied.'
|
|
elif sortType == 'alpha':
|
|
sortInformation = 'Words sorted alphabetically.'
|
|
elif sortType == 'alpha-black':
|
|
sortInformation = 'Words sorted alphabetically, with blacklist applied.'
|
|
cantoHeaders = '<h1>' + canticle + ' ' + cantoNum + '</h1>\n<h3>' + sortInformation + '</h3>\n'
|
|
boilerplate = '<html>\n' + '<head><meta charset="UTF-8">' + '<title>' + canticleAbbrv + ' ' + cantoNum + ' ' + sortType + '</title></head>\n' + cantoHeaders + '<table>\n<tr>\n<th>Word</th><th>Count</th>\n</tr>\n'
|
|
boilerclose = '</table>\n</html>'
|
|
htmlCanto.write(boilerplate)
|
|
for word, count in sortedDict.items():
|
|
htmlInput = '<tr>\n<td>' + word + '</td><td>' + str(count) + '</td>\n</tr>\n'
|
|
htmlCanto.write(htmlInput)
|
|
htmlCanto.write(boilerclose)
|
|
htmlCanto.close()
|
|
|
|
def makeDirs():
|
|
# moves *.csv to ../comedy-wordcount.d/csv/...
|
|
# moves *.html to ../comedy-wordcount.d/html/...
|
|
csvFiles = glob.iglob(os.path.join(cantoDir, '*.csv'))
|
|
htmlFiles = glob.iglob(os.path.join(cantoDir, '*.html'))
|
|
for csvFile in csvFiles:
|
|
canticle = csvFile.split('/')[-1].split('-')[1]
|
|
if re.search('num\.', csvFile):
|
|
numDir = '../../comedy-wordcount.d/csv/' + canticle + '-num/'
|
|
shutil.move(csvFile, numDir)
|
|
if re.search('num-black', csvFile):
|
|
numBlackDir = '../../comedy-wordcount.d/csv/' + canticle + '-num-black/'
|
|
shutil.move(csvFile, numBlackDir)
|
|
if re.search('alpha\.', csvFile):
|
|
alphaDir = '../../comedy-wordcount.d/csv/' + canticle + '-alpha/'
|
|
shutil.move(csvFile, alphaDir)
|
|
if re.search('alpha-black', csvFile):
|
|
alphaBlackDir = '../../comedy-wordcount.d/csv/' + canticle + '-alpha-black/'
|
|
shutil.move(csvFile, alphaBlackDir)
|
|
for htmlFile in htmlFiles:
|
|
canticle = htmlFile.split('/')[-1].split('-')[1]
|
|
if re.search('num\.', htmlFile):
|
|
numDir = '../../comedy-wordcount.d/html/' + canticle + '-num/'
|
|
shutil.move(htmlFile, numDir)
|
|
if re.search('num-black', htmlFile):
|
|
numBlackDir = '../../comedy-wordcount.d/html/' + canticle + '-num-black/'
|
|
shutil.move(htmlFile, numBlackDir)
|
|
if re.search('alpha\.', htmlFile):
|
|
alphaDir = '../../comedy-wordcount.d/html/' + canticle + '-alpha/'
|
|
shutil.move(htmlFile, alphaDir)
|
|
if re.search('alpha-black', htmlFile):
|
|
alphaBlackDir = '../../comedy-wordcount.d/html/' + canticle + '-alpha-black/'
|
|
shutil.move(htmlFile, alphaBlackDir)
|
|
|
|
def main():
|
|
for canto in os.listdir(cantoDir):
|
|
if re.search('\.xml$', canto):
|
|
test = 2
|
|
cantoFile = canto
|
|
wordCountDict = createWordCountDict(cantoFile)
|
|
|
|
# create sorted dicts
|
|
wordCountDictNumSorted = numSort(wordCountDict)
|
|
wordCountDictNumSortedBlack = blackSort(wordCountDictNumSorted)
|
|
wordCountDictAlphaSorted = alphaSort(wordCountDict)
|
|
wordCountDictAlphaSortedBlack = blackSort(wordCountDictAlphaSorted)
|
|
|
|
# create output files
|
|
dictToCsv(wordCountDictNumSorted, 'num', cantoFile) # `num`: sort numerically with no blacklist
|
|
dictToCsv(wordCountDictNumSortedBlack, 'num-black', cantoFile) # `num-black`: sort numerically with blacklist
|
|
dictToCsv(wordCountDictAlphaSorted, 'alpha', cantoFile)
|
|
dictToCsv(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile)
|
|
|
|
# remember to substitute in html entities for things like accents
|
|
dictToHtml(wordCountDictNumSorted, 'num', cantoFile)
|
|
dictToHtml(wordCountDictNumSortedBlack, 'num-black', cantoFile)
|
|
dictToHtml(wordCountDictAlphaSorted, 'alpha', cantoFile)
|
|
dictToHtml(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile)
|
|
|
|
makeDirs()
|
|
|
|
|
|
main()
|
|
|