lhess2021
4 years ago
4 changed files with 417 additions and 0 deletions
@ -0,0 +1,247 @@ |
|||||
|
import os |
||||
|
import re # find and replace |
||||
|
import shutil # move tmp file for overwrite |
||||
|
|
||||
|
cantoDir = os.path.abspath(os.getcwd()) |
||||
|
|
||||
|
def getXMLFileName(xmlCantoFile): |
||||
|
xmlCanto = xmlCantoFile.split('.')[0] + '.xml' |
||||
|
return xmlCanto |
||||
|
|
||||
|
def addWordTags(cantoFile): |
||||
|
""" |
||||
|
- [x] Parse filename to create xml file |
||||
|
- [x] Open file as one huge string (keep track of newlines) |
||||
|
- [x] Replace every space with a dummy char not used in the text, like `~` |
||||
|
- [ ] Traverse the big file-string from beginning to end |
||||
|
- [x] Define a "word-chars" group: letters, apostrofes |
||||
|
- [x] If you encounter a char that's a word-char |
||||
|
put a `<w>` in front of it and move to the next. |
||||
|
- [x] If the next char is also a word-char, move on |
||||
|
to the next because we're still in a word. |
||||
|
- [x] If the next char is not a word-char, |
||||
|
put a `</w>` in front of it and move to the next. |
||||
|
- [x] Replace all `~` with spaces and get back the newlines |
||||
|
- [x] Write to XML file |
||||
|
""" |
||||
|
xmlCanto = getXMLFileName(cantoFile) |
||||
|
with open(xmlCanto, 'w') as xmlCanto: |
||||
|
with open(cantoFile, 'r') as txtCanto: |
||||
|
originalText = txtCanto.read() |
||||
|
# Prepare canto for traversing |
||||
|
plainText = traversePreparation(originalText, '\n', '+', False) |
||||
|
plainText = traversePreparation(plainText, ' ', '~', False) |
||||
|
|
||||
|
# Define variables |
||||
|
wordChars = '[a-zA-ZÀ-ÿ0-9‘’]' |
||||
|
openWord = '<w>' |
||||
|
closeWord = '</w>' |
||||
|
insideWord = False |
||||
|
|
||||
|
# Traverse canto |
||||
|
for char in plainText: |
||||
|
if re.match(wordChars, char): |
||||
|
if insideWord == False: |
||||
|
xmlCanto.write(openWord) |
||||
|
xmlCanto.write(char) |
||||
|
insideWord = True |
||||
|
else: |
||||
|
xmlCanto.write(char) |
||||
|
else: |
||||
|
if char == '~': |
||||
|
if insideWord == True: |
||||
|
xmlCanto.write(closeWord + ' ') |
||||
|
insideWord = False |
||||
|
else: |
||||
|
xmlCanto.write(' ') |
||||
|
elif char == '+': |
||||
|
if insideWord == True: |
||||
|
xmlCanto.write(closeWord + '\n') # line ends with word |
||||
|
insideWord = False |
||||
|
else: |
||||
|
xmlCanto.write('\n') # line ends with punctuation |
||||
|
elif insideWord == True: |
||||
|
xmlCanto.write(closeWord + char) # punctuation right after to word |
||||
|
insideWord = False |
||||
|
else: |
||||
|
xmlCanto.write(char) # punctuation after punctuation |
||||
|
txtCanto.close() |
||||
|
xmlCanto.close() |
||||
|
|
||||
|
def traversePreparation(cantoText, find, replace, negate): |
||||
|
""" |
||||
|
Substitute find with replace. |
||||
|
If negate is True, substitute replace with find. |
||||
|
""" |
||||
|
if negate == False: |
||||
|
replacedText = cantoText.replace(find, replace) |
||||
|
else: |
||||
|
replacedText = cantoText.replace(replace, find) |
||||
|
return replacedText |
||||
|
|
||||
|
def addLineTags(xmlCantoFile): |
||||
|
""" |
||||
|
add <l l_num="n"> to the beginning of each line number n |
||||
|
add </l> to the end of each line |
||||
|
""" |
||||
|
xmlCanto = getXMLFileName(xmlCantoFile) |
||||
|
tmpFile = '.' + xmlCantoFile + '.tmp' |
||||
|
with open(xmlCanto, 'r') as xmlCanto: |
||||
|
with open(tmpFile, 'w') as tmpCanto: |
||||
|
i = 1 |
||||
|
for line in xmlCanto: |
||||
|
# replace '\n' in line with ' </l>\n' |
||||
|
newLine = re.sub('\n', ' </l>\n', line) |
||||
|
newLine = '<l l_num="' + str(i) + '"> ' + newLine |
||||
|
i += 1 |
||||
|
tmpCanto.write(newLine) |
||||
|
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
||||
|
|
||||
|
def addTercetTags(xmlCantoFile): |
||||
|
""" |
||||
|
add <tercet t_num="n"> ... </tercet> around each group of three lines until the end |
||||
|
""" |
||||
|
xmlCanto = getXMLFileName(xmlCantoFile) |
||||
|
tmpFile = '.' + xmlCantoFile + '.tmp' |
||||
|
with open(xmlCanto, 'r') as xmlCanto: |
||||
|
lastLine = xmlCanto.readlines()[-1] |
||||
|
lastLineNum = int(lastLine.split('"')[1]) |
||||
|
xmlCanto.seek(0) |
||||
|
with open(tmpFile, 'w') as tmpCanto: |
||||
|
i = 0 |
||||
|
j = 2 |
||||
|
for line in xmlCanto: |
||||
|
#print(line) |
||||
|
i += 1 |
||||
|
#print('new i: ' + str(i)) |
||||
|
if i == 1: |
||||
|
newLine = '<tercet t_num="1">\n' + line |
||||
|
tmpCanto.write(newLine) |
||||
|
#print('first line newline: ' + newLine) |
||||
|
elif i % 3 == 0: |
||||
|
tercetNum = str(j) |
||||
|
newLineSub = '\n</tercet>\n<tercet t_num="' + tercetNum + '">\n' |
||||
|
newLine = re.sub('\n', newLineSub, line) |
||||
|
tmpCanto.write(newLine) |
||||
|
j += 1 |
||||
|
#print('every 3 lines newline: ' + newLine) |
||||
|
elif i == lastLineNum: |
||||
|
newLine = re.sub('\n', '\n</tercet>', line) |
||||
|
tmpCanto.write(newLine) |
||||
|
#print('last line newline: ' + newLine) |
||||
|
else: |
||||
|
tmpCanto.write(line) |
||||
|
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
||||
|
|
||||
|
def wrapTags(tag, xmlCantoFile): |
||||
|
""" |
||||
|
used to add <tercets>, <canto>, and <text> tags around <canto> body |
||||
|
""" |
||||
|
xmlCanto = getXMLFileName(xmlCantoFile) |
||||
|
if tag == 'TEI': |
||||
|
openTag = '<TEI xmlns="http://www.tei-c.org/ns/1.0">\n' |
||||
|
else: |
||||
|
openTag = '<' + tag + '>\n' |
||||
|
closeTag = '\n</' + tag + '>' |
||||
|
with open(xmlCanto, 'r') as xmlCantoContents: |
||||
|
contents = xmlCantoContents.read() |
||||
|
with open(xmlCanto, 'w') as xmlCantoContents: |
||||
|
xmlCantoContents.write(openTag) |
||||
|
xmlCantoContents.write(contents) |
||||
|
xmlCantoContents.write(closeTag) |
||||
|
xmlCantoContents.close() |
||||
|
|
||||
|
def addTEIHeaderTags(xmlCantoFile): |
||||
|
""" |
||||
|
The following is the minimum required TEI header |
||||
|
<TEI xmlns="http://www.tei-c.org/ns/1.0"> |
||||
|
<fileDesc> |
||||
|
<titleStmt> |
||||
|
<title> |
||||
|
</title> |
||||
|
<author> |
||||
|
</author> |
||||
|
<respStmt> |
||||
|
</respStmt> |
||||
|
</titleStmt> |
||||
|
<publicationStmt> |
||||
|
<publisher> |
||||
|
</publisher> |
||||
|
</publicationStmt> |
||||
|
<sourceDesc> |
||||
|
<citation> |
||||
|
</citation> |
||||
|
</sourceDesc> |
||||
|
</fileDesc> |
||||
|
<text> |
||||
|
""" |
||||
|
xmlCanto = getXMLFileName(xmlCantoFile) |
||||
|
|
||||
|
def addSpaces(xmlCantoFile): |
||||
|
""" |
||||
|
Add a given number of spaces before each opening and closing tag. |
||||
|
User specifies the number of spaces only once, in a series of lists |
||||
|
which hold strings of each tag that needs spaces before it. This |
||||
|
function parses the names of the list to determine the number of |
||||
|
spaces to add before each tag (e.g. list2 => 2 spaces). It replaces |
||||
|
the tags in the document with the spaces affixed. |
||||
|
""" |
||||
|
# XML is hierarchical, so each "level" needs its own number of spaces |
||||
|
levelDict = { |
||||
|
'0' : ['TEI'], |
||||
|
'1' : ['teiHeader', 'text' ], |
||||
|
'2' : ['fileDesc' , 'canto' ], |
||||
|
'3' : ['titleStmt', 'tercets'], |
||||
|
'4' : [ 'tercet' ], |
||||
|
'5' : [ 'l' ] |
||||
|
} |
||||
|
|
||||
|
#levelDictLen = int(sorted(levelDict.keys())[-1]) + 1 # number of levels. add 1 to make range() work below |
||||
|
tags = ['TEI', 'teiHeader', 'text', 'fileDesc', 'canto', 'titleStmt', 'tercets', 'tercet', 'l' ] |
||||
|
|
||||
|
xmlCanto = getXMLFileName(xmlCantoFile) |
||||
|
tmpFile = '.' + xmlCantoFile + '.tmp' |
||||
|
|
||||
|
with open(xmlCantoFile, 'r') as xmlCanto: |
||||
|
with open(tmpFile, 'w') as tmpCanto: |
||||
|
for line in xmlCanto: |
||||
|
for tag in tags: |
||||
|
# need open tag regex b/c we don't know if tag has attributes, |
||||
|
# and if we did, we might not know their value |
||||
|
openTagRegex = '(<' + tag + ')(>|\s.*>)' # match <tag> and <tag attr="value"> but not <tags> |
||||
|
closeTag = '</' + tag + '>' |
||||
|
for level in levelDict.keys(): |
||||
|
if tag in levelDict[level]: |
||||
|
spaces = ' ' * int(level) |
||||
|
if re.search(openTagRegex, line): |
||||
|
newLine = re.sub(openTagRegex, spaces + r"\1\2", line) |
||||
|
tmpCanto.write(newLine) |
||||
|
elif closeTag in line: |
||||
|
if tag != 'l': # don't add spaces to closing line tags |
||||
|
newLine = re.sub(closeTag, spaces + closeTag, line) |
||||
|
tmpCanto.write(newLine) |
||||
|
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
||||
|
|
||||
|
def closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto): |
||||
|
tmpCanto.close() |
||||
|
xmlCanto.close() |
||||
|
shutil.copy(tmpFile, xmlCantoFile) |
||||
|
os.remove(tmpFile) |
||||
|
|
||||
|
def main(): |
||||
|
for cantoFile in os.listdir(cantoDir): |
||||
|
xmlCantoName = getXMLFileName(cantoFile) |
||||
|
if re.search('\.txt$', cantoFile): |
||||
|
addWordTags(cantoFile) |
||||
|
addLineTags(xmlCantoName) |
||||
|
addTercetTags(xmlCantoName) |
||||
|
wrapTags('tercets', xmlCantoName) |
||||
|
wrapTags('canto', xmlCantoName) |
||||
|
wrapTags('text', xmlCantoName) |
||||
|
#addTEIHeaderTags() |
||||
|
wrapTags('TEI', xmlCantoName) |
||||
|
addSpaces(xmlCantoName) |
||||
|
os.remove(cantoFile) # don't delete txt files while debugging so you don't have to keep copying them |
||||
|
|
||||
|
|
||||
|
main() |
||||
@ -0,0 +1,24 @@ |
|||||
|
import shutil |
||||
|
import os |
||||
|
import re |
||||
|
|
||||
|
cantoDir = os.path.abspath(os.getcwd()) |
||||
|
|
||||
|
def main(): |
||||
|
""" |
||||
|
Take txt file from `dante-visualized` project and remove blank lines |
||||
|
""" |
||||
|
for txtFile in os.listdir(cantoDir): |
||||
|
if re.search('\.txt$', txtFile): |
||||
|
tmpFile = '.' + txtFile+ '.tmp' |
||||
|
with open(txtFile, 'r') as txt: |
||||
|
with open(tmpFile, 'w') as tmp: |
||||
|
for line in txt: |
||||
|
if line.rstrip(): |
||||
|
tmp.write(line) |
||||
|
tmp.close() |
||||
|
txt.close() |
||||
|
shutil.copy(tmpFile, txtFile) |
||||
|
os.remove(tmpFile) |
||||
|
|
||||
|
main() |
||||
@ -0,0 +1,22 @@ |
|||||
|
import re |
||||
|
import os |
||||
|
|
||||
|
canto_dir = os.path.abspath(os.getcwd()) |
||||
|
canto_files = os.listdir(canto_dir) |
||||
|
|
||||
|
def rename_canto(canto): |
||||
|
""" |
||||
|
Take txt canto files from `dante-visualized` and change the name styling |
||||
|
""" |
||||
|
# parse canto name |
||||
|
# put it back together |
||||
|
print('canto name: ' + canto) |
||||
|
new_canto = re.sub('\_','-',canto) |
||||
|
new_canto = re.sub('Canto[IVX]*-','',new_canto) |
||||
|
new_canto = new_canto.lower() |
||||
|
print('--> new canto name: ' + new_canto) |
||||
|
os.rename(canto,new_canto) |
||||
|
|
||||
|
for canto in canto_files: |
||||
|
if canto != 'rename-txt.py' and canto != '.rename-txt.py.swp': |
||||
|
rename_canto(canto) |
||||
@ -0,0 +1,124 @@ |
|||||
|
import re |
||||
|
import os |
||||
|
import xml.etree.ElementTree as ET |
||||
|
|
||||
|
""" |
||||
|
This script takes xml canto files and counts the number of unique words as |
||||
|
defined by the value of `<w>` nodes. |
||||
|
|
||||
|
8 files are generated: csv and html of |
||||
|
- sorted numerically and alphabetically |
||||
|
- sorted numerically and alphabetically with blacklist applied |
||||
|
""" |
||||
|
|
||||
|
cantoDir = os.path.abspath(os.getcwd()) |
||||
|
|
||||
|
blacklist = ('e', 'che', 'la', 'di', 'a', 'per', '’l', 'mi', 'tu', 'con', |
||||
|
'lo', 'sì', 'sua', 'li', 'del', 'si', 'nel', 'era', 'ch’i’', 'è', |
||||
|
'là', 'le', 'de', 'quella', 'poi', 'ma', 'il', 'già', '’n', 'una', |
||||
|
'tanto', 'se’', 'quel', 'qual', 'perché', 'in', 'come', 'al', 'da', |
||||
|
'un', 'i', 'ti', 'a', 'per', 'con', 'lo', 'al', 'poi', 'E', 'il') |
||||
|
|
||||
|
def createWordCountDict(cantoFile): |
||||
|
""" |
||||
|
for node in xpathQueryAllWordNodes: |
||||
|
if xpathValue(node) in wordCountDict(keys): |
||||
|
wordCountDict[node] += 1 |
||||
|
else: |
||||
|
wordCountDict[xpathValue(node)] = 1 |
||||
|
""" |
||||
|
wordCountDict = dict() |
||||
|
tree = ET.parse(cantoFile) |
||||
|
root = tree.getroot() |
||||
|
# TODO |
||||
|
words = root.findall(".//{http://www.tei-c.org/ns/1.0}w") |
||||
|
for word in words: |
||||
|
if word.text in wordCountDict: |
||||
|
wordCountDict[word.text] += 1 |
||||
|
else: |
||||
|
wordCountDict[word.text] = 1 |
||||
|
return wordCountDict |
||||
|
|
||||
|
def numSort(wordCountDict): |
||||
|
# Need reverse = True to sort greatest to least |
||||
|
wordCountDictNumSorted = dict(sorted(wordCountDict.items(), key=lambda item: item[1], reverse = True)) |
||||
|
return wordCountDictNumSorted |
||||
|
|
||||
|
def alphaSort(wordCountDict): |
||||
|
# Capital letters come first with this implementation |
||||
|
wordCountDictAlphaSorted = dict(sorted(wordCountDict.items())) |
||||
|
return wordCountDictAlphaSorted |
||||
|
|
||||
|
def blackSort(wordCountDictSorted): |
||||
|
sortedCopy = dict(wordCountDictSorted) |
||||
|
for word in wordCountDictSorted: |
||||
|
if word in blacklist: |
||||
|
del sortedCopy[word] |
||||
|
return sortedCopy |
||||
|
|
||||
|
def dictToCsv(sortedDict, sortType, cantoFile): |
||||
|
""" |
||||
|
write a sorted dict to a CSV file for that canto |
||||
|
name it based on sorting type (numerical or alphabetical) |
||||
|
abbreviate sortType as `num` or `alpha` |
||||
|
""" |
||||
|
csvCanto = cantoFile.split('.')[0] + '-' + sortType + '.csv' |
||||
|
cantoNum = str(int(cantoFile.split('-')[0])) |
||||
|
canticle = cantoFile.split('-')[1] |
||||
|
|
||||
|
with open(csvCanto, 'w') as csvCanto: |
||||
|
cantoInformation = canticle + ',' + cantoNum + '\n' |
||||
|
csvCanto.write(cantoInformation) |
||||
|
for word, count in sortedDict.items(): |
||||
|
csvInput = word + ',' + str(count) + '\n' |
||||
|
csvCanto.write(csvInput) |
||||
|
csvCanto.close() |
||||
|
|
||||
|
def dictToHtml(sortedDict, sortType, cantoFile): |
||||
|
htmlCanto = cantoFile.split('.')[0] + '-' + sortType + '.html' |
||||
|
with open(htmlCanto, 'w') as htmlCanto: |
||||
|
cantoNum = str(int(cantoFile.split('-')[0])) |
||||
|
canticle = cantoFile.split('-')[1].title() |
||||
|
if sortType == 'num': |
||||
|
sortInformation = 'Words sorted by most to least occurrences.' |
||||
|
elif sortType == 'num-black': |
||||
|
sortInformation = 'Words sorted by most to least occurrences, with blacklist applied.' |
||||
|
elif sortType == 'alpha': |
||||
|
sortInformation = 'Words sorted alphabetically.' |
||||
|
elif sortType == 'alpha-black': |
||||
|
sortInformation = 'Words sorted alphabetically, with blacklist applied.' |
||||
|
cantoHeaders = '<h1>' + canticle + ' ' + cantoNum + '</h1>\n<h3>' + sortInformation + '</h3>\n' |
||||
|
boilerplate = '<html>\n' + cantoHeaders + '<table>\n<tr>\n<th>Word</th><th>Count</th>\n</tr>\n' |
||||
|
boilerclose = '</table>\n</html>' |
||||
|
htmlCanto.write(boilerplate) |
||||
|
for word, count in sortedDict.items(): |
||||
|
htmlInput = '<tr>\n<td>' + word + '</td><td>' + str(count) + '</td>\n</tr>\n' |
||||
|
htmlCanto.write(htmlInput) |
||||
|
htmlCanto.write(boilerclose) |
||||
|
htmlCanto.close() |
||||
|
|
||||
|
def main(): |
||||
|
for canto in os.listdir(cantoDir): |
||||
|
if re.search('\.xml$', canto): |
||||
|
cantoFile = canto |
||||
|
wordCountDict = createWordCountDict(cantoFile) |
||||
|
|
||||
|
# create sorted dicts |
||||
|
wordCountDictNumSorted = numSort(wordCountDict) |
||||
|
wordCountDictNumSortedBlack = blackSort(wordCountDictNumSorted) |
||||
|
wordCountDictAlphaSorted = alphaSort(wordCountDict) |
||||
|
wordCountDictAlphaSortedBlack = blackSort(wordCountDictAlphaSorted) |
||||
|
|
||||
|
# create output files |
||||
|
dictToCsv(wordCountDictNumSorted, 'num', cantoFile) # `num-full`: sort numerically with no blacklist |
||||
|
dictToCsv(wordCountDictNumSortedBlack, 'num-black', cantoFile) # `num`: sort numerically with blacklist |
||||
|
dictToCsv(wordCountDictAlphaSorted, 'alpha', cantoFile) |
||||
|
dictToCsv(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) |
||||
|
|
||||
|
dictToHtml(wordCountDictNumSorted, 'num', cantoFile) |
||||
|
dictToHtml(wordCountDictNumSortedBlack, 'num-black', cantoFile) |
||||
|
dictToHtml(wordCountDictAlphaSorted, 'alpha', cantoFile) |
||||
|
dictToHtml(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) |
||||
|
|
||||
|
|
||||
|
main() |
||||
Loading…
Reference in new issue