You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
124 lines
5.1 KiB
124 lines
5.1 KiB
import re
|
|
import os
|
|
import xml.etree.ElementTree as ET
|
|
|
|
"""
|
|
This script takes xml canto files and counts the number of unique words as
|
|
defined by the value of `<w>` nodes.
|
|
|
|
8 files are generated: csv and html of
|
|
- sorted numerically and alphabetically
|
|
- sorted numerically and alphabetically with blacklist applied
|
|
"""
|
|
|
|
cantoDir = os.path.abspath(os.getcwd())
|
|
|
|
blacklist = ('e', 'che', 'la', 'di', 'a', 'per', '’l', 'mi', 'tu', 'con',
|
|
'lo', 'sì', 'sua', 'li', 'del', 'si', 'nel', 'era', 'ch’i’', 'è',
|
|
'là', 'le', 'de', 'quella', 'poi', 'ma', 'il', 'già', '’n', 'una',
|
|
'tanto', 'se’', 'quel', 'qual', 'perché', 'in', 'come', 'al', 'da',
|
|
'un', 'i', 'ti', 'a', 'per', 'con', 'lo', 'al', 'poi', 'E', 'il')
|
|
|
|
def createWordCountDict(cantoFile):
|
|
"""
|
|
for node in xpathQueryAllWordNodes:
|
|
if xpathValue(node) in wordCountDict(keys):
|
|
wordCountDict[node] += 1
|
|
else:
|
|
wordCountDict[xpathValue(node)] = 1
|
|
"""
|
|
wordCountDict = dict()
|
|
tree = ET.parse(cantoFile)
|
|
root = tree.getroot()
|
|
# TODO
|
|
words = root.findall(".//{http://www.tei-c.org/ns/1.0}w")
|
|
for word in words:
|
|
if word.text in wordCountDict:
|
|
wordCountDict[word.text] += 1
|
|
else:
|
|
wordCountDict[word.text] = 1
|
|
return wordCountDict
|
|
|
|
def numSort(wordCountDict):
|
|
# Need reverse = True to sort greatest to least
|
|
wordCountDictNumSorted = dict(sorted(wordCountDict.items(), key=lambda item: item[1], reverse = True))
|
|
return wordCountDictNumSorted
|
|
|
|
def alphaSort(wordCountDict):
|
|
# Capital letters come first with this implementation
|
|
wordCountDictAlphaSorted = dict(sorted(wordCountDict.items()))
|
|
return wordCountDictAlphaSorted
|
|
|
|
def blackSort(wordCountDictSorted):
|
|
sortedCopy = dict(wordCountDictSorted)
|
|
for word in wordCountDictSorted:
|
|
if word in blacklist:
|
|
del sortedCopy[word]
|
|
return sortedCopy
|
|
|
|
def dictToCsv(sortedDict, sortType, cantoFile):
|
|
"""
|
|
write a sorted dict to a CSV file for that canto
|
|
name it based on sorting type (numerical or alphabetical)
|
|
abbreviate sortType as `num` or `alpha`
|
|
"""
|
|
csvCanto = cantoFile.split('.')[0] + '-' + sortType + '.csv'
|
|
cantoNum = str(int(cantoFile.split('-')[0]))
|
|
canticle = cantoFile.split('-')[1]
|
|
|
|
with open(csvCanto, 'w') as csvCanto:
|
|
cantoInformation = canticle + ',' + cantoNum + '\n'
|
|
csvCanto.write(cantoInformation)
|
|
for word, count in sortedDict.items():
|
|
csvInput = word + ',' + str(count) + '\n'
|
|
csvCanto.write(csvInput)
|
|
csvCanto.close()
|
|
|
|
def dictToHtml(sortedDict, sortType, cantoFile):
|
|
htmlCanto = cantoFile.split('.')[0] + '-' + sortType + '.html'
|
|
with open(htmlCanto, 'w') as htmlCanto:
|
|
cantoNum = str(int(cantoFile.split('-')[0]))
|
|
canticle = cantoFile.split('-')[1].title()
|
|
if sortType == 'num':
|
|
sortInformation = 'Words sorted by most to least occurrences.'
|
|
elif sortType == 'num-black':
|
|
sortInformation = 'Words sorted by most to least occurrences, with blacklist applied.'
|
|
elif sortType == 'alpha':
|
|
sortInformation = 'Words sorted alphabetically.'
|
|
elif sortType == 'alpha-black':
|
|
sortInformation = 'Words sorted alphabetically, with blacklist applied.'
|
|
cantoHeaders = '<h1>' + canticle + ' ' + cantoNum + '</h1>\n<h3>' + sortInformation + '</h3>\n'
|
|
boilerplate = '<html>\n' + cantoHeaders + '<table>\n<tr>\n<th>Word</th><th>Count</th>\n</tr>\n'
|
|
boilerclose = '</table>\n</html>'
|
|
htmlCanto.write(boilerplate)
|
|
for word, count in sortedDict.items():
|
|
htmlInput = '<tr>\n<td>' + word + '</td><td>' + str(count) + '</td>\n</tr>\n'
|
|
htmlCanto.write(htmlInput)
|
|
htmlCanto.write(boilerclose)
|
|
htmlCanto.close()
|
|
|
|
def main():
|
|
for canto in os.listdir(cantoDir):
|
|
if re.search('\.xml$', canto):
|
|
cantoFile = canto
|
|
wordCountDict = createWordCountDict(cantoFile)
|
|
|
|
# create sorted dicts
|
|
wordCountDictNumSorted = numSort(wordCountDict)
|
|
wordCountDictNumSortedBlack = blackSort(wordCountDictNumSorted)
|
|
wordCountDictAlphaSorted = alphaSort(wordCountDict)
|
|
wordCountDictAlphaSortedBlack = blackSort(wordCountDictAlphaSorted)
|
|
|
|
# create output files
|
|
dictToCsv(wordCountDictNumSorted, 'num', cantoFile) # `num-full`: sort numerically with no blacklist
|
|
dictToCsv(wordCountDictNumSortedBlack, 'num-black', cantoFile) # `num`: sort numerically with blacklist
|
|
dictToCsv(wordCountDictAlphaSorted, 'alpha', cantoFile)
|
|
dictToCsv(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile)
|
|
|
|
dictToHtml(wordCountDictNumSorted, 'num', cantoFile)
|
|
dictToHtml(wordCountDictNumSortedBlack, 'num-black', cantoFile)
|
|
dictToHtml(wordCountDictAlphaSorted, 'alpha', cantoFile)
|
|
dictToHtml(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile)
|
|
|
|
|
|
main()
|
|
|