import re import os import xml.etree.ElementTree as ET import glob import shutil """ This script takes xml canto files and counts the number of unique words as defined by the value of `` nodes. 8 files are generated: csv and html of cantos - sorted numerically and alphabetically - sorted numerically and alphabetically with blacklist applied """ cantoDir = os.path.abspath(os.getcwd()) blacklist = ('e', 'che', 'la', 'di', 'a', 'per', '’l', 'mi', 'tu', 'con', 'lo', 'sì', 'sua', 'li', 'del', 'si', 'nel', 'era', 'ch’i’', 'è', 'là', 'le', 'de', 'quella', 'poi', 'ma', 'il', 'già', '’n', 'una', 'tanto', 'se’', 'quel', 'qual', 'perché', 'in', 'come', 'al', 'da', 'un', 'i', 'ti', 'a', 'per', 'con', 'lo', 'al', 'poi', 'E', 'il') def createWordCountDict(cantoFile): """ for node in xpathQueryAllWordNodes: if xpathValue(node) in wordCountDict(keys): wordCountDict[node] += 1 else: wordCountDict[xpathValue(node)] = 1 """ wordCountDict = dict() print(cantoFile) tree = ET.parse(cantoFile) root = tree.getroot() # TODO words = root.findall(".//{http://www.tei-c.org/ns/1.0}w") for word in words: if word.text in wordCountDict: wordCountDict[word.text] += 1 else: wordCountDict[word.text] = 1 return wordCountDict def numSort(wordCountDict): # Need reverse = True to sort greatest to least wordCountDictNumSorted = dict(sorted(wordCountDict.items(), key=lambda item: item[1], reverse = True)) return wordCountDictNumSorted def alphaSort(wordCountDict): # Capital letters come first with this implementation wordCountDictAlphaSorted = dict(sorted(wordCountDict.items())) return wordCountDictAlphaSorted def blackSort(wordCountDictSorted): sortedCopy = dict(wordCountDictSorted) for word in wordCountDictSorted: if word in blacklist: del sortedCopy[word] return sortedCopy def dictToCsv(sortedDict, sortType, cantoFile): """ write a sorted dict to a CSV file for that canto name it based on sorting type (numerical or alphabetical) abbreviate sortType as `num` or `alpha` """ csvCanto = cantoFile.split('.')[0] + '-' + sortType + '.csv' cantoNum = str(int(cantoFile.split('-')[0])) canticle = cantoFile.split('-')[1] with open(csvCanto, 'w') as csvCanto: cantoInformation = canticle + ',' + cantoNum + '\n' csvCanto.write(cantoInformation) for word, count in sortedDict.items(): csvInput = word + ',' + str(count) + '\n' csvCanto.write(csvInput) csvCanto.close() def dictToHtml(sortedDict, sortType, cantoFile): htmlCanto = cantoFile.split('.')[0] + '-' + sortType + '.html' with open(htmlCanto, 'w') as htmlCanto: cantoNum = str(int(cantoFile.split('-')[0])) canticle = cantoFile.split('-')[1].title() if sortType == 'num': sortInformation = 'Words sorted by most to least occurrences.' elif sortType == 'num-black': sortInformation = 'Words sorted by most to least occurrences, with blacklist applied.' elif sortType == 'alpha': sortInformation = 'Words sorted alphabetically.' elif sortType == 'alpha-black': sortInformation = 'Words sorted alphabetically, with blacklist applied.' cantoHeaders = '

' + canticle + ' ' + cantoNum + '

\n

' + sortInformation + '

\n' boilerplate = '\n' + cantoHeaders + '\n\n\n\n' boilerclose = '
WordCount
\n' htmlCanto.write(boilerplate) for word, count in sortedDict.items(): htmlInput = '\n' + word + '' + str(count) + '\n\n' htmlCanto.write(htmlInput) htmlCanto.write(boilerclose) htmlCanto.close() def makeDirs(): # moves *.csv to ../comedy-wordcount.d/csv/... # moves *.html to ../comedy-wordcount.d/html/... csvFiles = glob.iglob(os.path.join(cantoDir, '*.csv')) htmlFiles = glob.iglob(os.path.join(cantoDir, '*.html')) for csvFile in csvFiles: canticle = csvFile.split('/')[-1].split('-')[1] if re.search('num\.', csvFile): numDir = '../../comedy-wordcount.d/csv/' + canticle + '-num/' shutil.move(csvFile, numDir) if re.search('num-black', csvFile): numBlackDir = '../../comedy-wordcount.d/csv/' + canticle + '-num-black/' shutil.move(csvFile, numBlackDir) if re.search('alpha\.', csvFile): alphaDir = '../../comedy-wordcount.d/csv/' + canticle + '-alpha/' shutil.move(csvFile, alphaDir) if re.search('alpha-black', csvFile): alphaBlackDir = '../../comedy-wordcount.d/csv/' + canticle + '-alpha-black/' shutil.move(csvFile, alphaBlackDir) for htmlFile in htmlFiles: canticle = htmlFile.split('/')[-1].split('-')[1] if re.search('num\.', htmlFile): numDir = '../../comedy-wordcount.d/html/' + canticle + '-num/' shutil.move(htmlFile, numDir) if re.search('num-black', htmlFile): numBlackDir = '../../comedy-wordcount.d/html/' + canticle + '-num-black/' shutil.move(htmlFile, numBlackDir) if re.search('alpha\.', htmlFile): alphaDir = '../../comedy-wordcount.d/html/' + canticle + '-alpha/' shutil.move(htmlFile, alphaDir) if re.search('alpha-black', htmlFile): alphaBlackDir = '../../comedy-wordcount.d/html/' + canticle + '-alpha-black/' shutil.move(htmlFile, alphaBlackDir) def main(): for canto in os.listdir(cantoDir): if re.search('\.xml$', canto): cantoFile = canto wordCountDict = createWordCountDict(cantoFile) # create sorted dicts wordCountDictNumSorted = numSort(wordCountDict) wordCountDictNumSortedBlack = blackSort(wordCountDictNumSorted) wordCountDictAlphaSorted = alphaSort(wordCountDict) wordCountDictAlphaSortedBlack = blackSort(wordCountDictAlphaSorted) # create output files dictToCsv(wordCountDictNumSorted, 'num', cantoFile) # `num-full`: sort numerically with no blacklist dictToCsv(wordCountDictNumSortedBlack, 'num-black', cantoFile) # `num`: sort numerically with blacklist dictToCsv(wordCountDictAlphaSorted, 'alpha', cantoFile) dictToCsv(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) dictToHtml(wordCountDictNumSorted, 'num', cantoFile) dictToHtml(wordCountDictNumSortedBlack, 'num-black', cantoFile) dictToHtml(wordCountDictAlphaSorted, 'alpha', cantoFile) dictToHtml(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) makeDirs() main()