lhess2021
4 years ago
4 changed files with 417 additions and 0 deletions
@ -0,0 +1,247 @@ |
|||
import os |
|||
import re # find and replace |
|||
import shutil # move tmp file for overwrite |
|||
|
|||
cantoDir = os.path.abspath(os.getcwd()) |
|||
|
|||
def getXMLFileName(xmlCantoFile): |
|||
xmlCanto = xmlCantoFile.split('.')[0] + '.xml' |
|||
return xmlCanto |
|||
|
|||
def addWordTags(cantoFile): |
|||
""" |
|||
- [x] Parse filename to create xml file |
|||
- [x] Open file as one huge string (keep track of newlines) |
|||
- [x] Replace every space with a dummy char not used in the text, like `~` |
|||
- [ ] Traverse the big file-string from beginning to end |
|||
- [x] Define a "word-chars" group: letters, apostrofes |
|||
- [x] If you encounter a char that's a word-char |
|||
put a `<w>` in front of it and move to the next. |
|||
- [x] If the next char is also a word-char, move on |
|||
to the next because we're still in a word. |
|||
- [x] If the next char is not a word-char, |
|||
put a `</w>` in front of it and move to the next. |
|||
- [x] Replace all `~` with spaces and get back the newlines |
|||
- [x] Write to XML file |
|||
""" |
|||
xmlCanto = getXMLFileName(cantoFile) |
|||
with open(xmlCanto, 'w') as xmlCanto: |
|||
with open(cantoFile, 'r') as txtCanto: |
|||
originalText = txtCanto.read() |
|||
# Prepare canto for traversing |
|||
plainText = traversePreparation(originalText, '\n', '+', False) |
|||
plainText = traversePreparation(plainText, ' ', '~', False) |
|||
|
|||
# Define variables |
|||
wordChars = '[a-zA-ZÀ-ÿ0-9‘’]' |
|||
openWord = '<w>' |
|||
closeWord = '</w>' |
|||
insideWord = False |
|||
|
|||
# Traverse canto |
|||
for char in plainText: |
|||
if re.match(wordChars, char): |
|||
if insideWord == False: |
|||
xmlCanto.write(openWord) |
|||
xmlCanto.write(char) |
|||
insideWord = True |
|||
else: |
|||
xmlCanto.write(char) |
|||
else: |
|||
if char == '~': |
|||
if insideWord == True: |
|||
xmlCanto.write(closeWord + ' ') |
|||
insideWord = False |
|||
else: |
|||
xmlCanto.write(' ') |
|||
elif char == '+': |
|||
if insideWord == True: |
|||
xmlCanto.write(closeWord + '\n') # line ends with word |
|||
insideWord = False |
|||
else: |
|||
xmlCanto.write('\n') # line ends with punctuation |
|||
elif insideWord == True: |
|||
xmlCanto.write(closeWord + char) # punctuation right after to word |
|||
insideWord = False |
|||
else: |
|||
xmlCanto.write(char) # punctuation after punctuation |
|||
txtCanto.close() |
|||
xmlCanto.close() |
|||
|
|||
def traversePreparation(cantoText, find, replace, negate): |
|||
""" |
|||
Substitute find with replace. |
|||
If negate is True, substitute replace with find. |
|||
""" |
|||
if negate == False: |
|||
replacedText = cantoText.replace(find, replace) |
|||
else: |
|||
replacedText = cantoText.replace(replace, find) |
|||
return replacedText |
|||
|
|||
def addLineTags(xmlCantoFile): |
|||
""" |
|||
add <l l_num="n"> to the beginning of each line number n |
|||
add </l> to the end of each line |
|||
""" |
|||
xmlCanto = getXMLFileName(xmlCantoFile) |
|||
tmpFile = '.' + xmlCantoFile + '.tmp' |
|||
with open(xmlCanto, 'r') as xmlCanto: |
|||
with open(tmpFile, 'w') as tmpCanto: |
|||
i = 1 |
|||
for line in xmlCanto: |
|||
# replace '\n' in line with ' </l>\n' |
|||
newLine = re.sub('\n', ' </l>\n', line) |
|||
newLine = '<l l_num="' + str(i) + '"> ' + newLine |
|||
i += 1 |
|||
tmpCanto.write(newLine) |
|||
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
|||
|
|||
def addTercetTags(xmlCantoFile): |
|||
""" |
|||
add <tercet t_num="n"> ... </tercet> around each group of three lines until the end |
|||
""" |
|||
xmlCanto = getXMLFileName(xmlCantoFile) |
|||
tmpFile = '.' + xmlCantoFile + '.tmp' |
|||
with open(xmlCanto, 'r') as xmlCanto: |
|||
lastLine = xmlCanto.readlines()[-1] |
|||
lastLineNum = int(lastLine.split('"')[1]) |
|||
xmlCanto.seek(0) |
|||
with open(tmpFile, 'w') as tmpCanto: |
|||
i = 0 |
|||
j = 2 |
|||
for line in xmlCanto: |
|||
#print(line) |
|||
i += 1 |
|||
#print('new i: ' + str(i)) |
|||
if i == 1: |
|||
newLine = '<tercet t_num="1">\n' + line |
|||
tmpCanto.write(newLine) |
|||
#print('first line newline: ' + newLine) |
|||
elif i % 3 == 0: |
|||
tercetNum = str(j) |
|||
newLineSub = '\n</tercet>\n<tercet t_num="' + tercetNum + '">\n' |
|||
newLine = re.sub('\n', newLineSub, line) |
|||
tmpCanto.write(newLine) |
|||
j += 1 |
|||
#print('every 3 lines newline: ' + newLine) |
|||
elif i == lastLineNum: |
|||
newLine = re.sub('\n', '\n</tercet>', line) |
|||
tmpCanto.write(newLine) |
|||
#print('last line newline: ' + newLine) |
|||
else: |
|||
tmpCanto.write(line) |
|||
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
|||
|
|||
def wrapTags(tag, xmlCantoFile): |
|||
""" |
|||
used to add <tercets>, <canto>, and <text> tags around <canto> body |
|||
""" |
|||
xmlCanto = getXMLFileName(xmlCantoFile) |
|||
if tag == 'TEI': |
|||
openTag = '<TEI xmlns="http://www.tei-c.org/ns/1.0">\n' |
|||
else: |
|||
openTag = '<' + tag + '>\n' |
|||
closeTag = '\n</' + tag + '>' |
|||
with open(xmlCanto, 'r') as xmlCantoContents: |
|||
contents = xmlCantoContents.read() |
|||
with open(xmlCanto, 'w') as xmlCantoContents: |
|||
xmlCantoContents.write(openTag) |
|||
xmlCantoContents.write(contents) |
|||
xmlCantoContents.write(closeTag) |
|||
xmlCantoContents.close() |
|||
|
|||
def addTEIHeaderTags(xmlCantoFile): |
|||
""" |
|||
The following is the minimum required TEI header |
|||
<TEI xmlns="http://www.tei-c.org/ns/1.0"> |
|||
<fileDesc> |
|||
<titleStmt> |
|||
<title> |
|||
</title> |
|||
<author> |
|||
</author> |
|||
<respStmt> |
|||
</respStmt> |
|||
</titleStmt> |
|||
<publicationStmt> |
|||
<publisher> |
|||
</publisher> |
|||
</publicationStmt> |
|||
<sourceDesc> |
|||
<citation> |
|||
</citation> |
|||
</sourceDesc> |
|||
</fileDesc> |
|||
<text> |
|||
""" |
|||
xmlCanto = getXMLFileName(xmlCantoFile) |
|||
|
|||
def addSpaces(xmlCantoFile): |
|||
""" |
|||
Add a given number of spaces before each opening and closing tag. |
|||
User specifies the number of spaces only once, in a series of lists |
|||
which hold strings of each tag that needs spaces before it. This |
|||
function parses the names of the list to determine the number of |
|||
spaces to add before each tag (e.g. list2 => 2 spaces). It replaces |
|||
the tags in the document with the spaces affixed. |
|||
""" |
|||
# XML is hierarchical, so each "level" needs its own number of spaces |
|||
levelDict = { |
|||
'0' : ['TEI'], |
|||
'1' : ['teiHeader', 'text' ], |
|||
'2' : ['fileDesc' , 'canto' ], |
|||
'3' : ['titleStmt', 'tercets'], |
|||
'4' : [ 'tercet' ], |
|||
'5' : [ 'l' ] |
|||
} |
|||
|
|||
#levelDictLen = int(sorted(levelDict.keys())[-1]) + 1 # number of levels. add 1 to make range() work below |
|||
tags = ['TEI', 'teiHeader', 'text', 'fileDesc', 'canto', 'titleStmt', 'tercets', 'tercet', 'l' ] |
|||
|
|||
xmlCanto = getXMLFileName(xmlCantoFile) |
|||
tmpFile = '.' + xmlCantoFile + '.tmp' |
|||
|
|||
with open(xmlCantoFile, 'r') as xmlCanto: |
|||
with open(tmpFile, 'w') as tmpCanto: |
|||
for line in xmlCanto: |
|||
for tag in tags: |
|||
# need open tag regex b/c we don't know if tag has attributes, |
|||
# and if we did, we might not know their value |
|||
openTagRegex = '(<' + tag + ')(>|\s.*>)' # match <tag> and <tag attr="value"> but not <tags> |
|||
closeTag = '</' + tag + '>' |
|||
for level in levelDict.keys(): |
|||
if tag in levelDict[level]: |
|||
spaces = ' ' * int(level) |
|||
if re.search(openTagRegex, line): |
|||
newLine = re.sub(openTagRegex, spaces + r"\1\2", line) |
|||
tmpCanto.write(newLine) |
|||
elif closeTag in line: |
|||
if tag != 'l': # don't add spaces to closing line tags |
|||
newLine = re.sub(closeTag, spaces + closeTag, line) |
|||
tmpCanto.write(newLine) |
|||
closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto) |
|||
|
|||
def closeCopyRemoveTmp(tmpFile, tmpCanto, xmlCantoFile, xmlCanto): |
|||
tmpCanto.close() |
|||
xmlCanto.close() |
|||
shutil.copy(tmpFile, xmlCantoFile) |
|||
os.remove(tmpFile) |
|||
|
|||
def main(): |
|||
for cantoFile in os.listdir(cantoDir): |
|||
xmlCantoName = getXMLFileName(cantoFile) |
|||
if re.search('\.txt$', cantoFile): |
|||
addWordTags(cantoFile) |
|||
addLineTags(xmlCantoName) |
|||
addTercetTags(xmlCantoName) |
|||
wrapTags('tercets', xmlCantoName) |
|||
wrapTags('canto', xmlCantoName) |
|||
wrapTags('text', xmlCantoName) |
|||
#addTEIHeaderTags() |
|||
wrapTags('TEI', xmlCantoName) |
|||
addSpaces(xmlCantoName) |
|||
os.remove(cantoFile) # don't delete txt files while debugging so you don't have to keep copying them |
|||
|
|||
|
|||
main() |
|||
@ -0,0 +1,24 @@ |
|||
import shutil |
|||
import os |
|||
import re |
|||
|
|||
cantoDir = os.path.abspath(os.getcwd()) |
|||
|
|||
def main(): |
|||
""" |
|||
Take txt file from `dante-visualized` project and remove blank lines |
|||
""" |
|||
for txtFile in os.listdir(cantoDir): |
|||
if re.search('\.txt$', txtFile): |
|||
tmpFile = '.' + txtFile+ '.tmp' |
|||
with open(txtFile, 'r') as txt: |
|||
with open(tmpFile, 'w') as tmp: |
|||
for line in txt: |
|||
if line.rstrip(): |
|||
tmp.write(line) |
|||
tmp.close() |
|||
txt.close() |
|||
shutil.copy(tmpFile, txtFile) |
|||
os.remove(tmpFile) |
|||
|
|||
main() |
|||
@ -0,0 +1,22 @@ |
|||
import re |
|||
import os |
|||
|
|||
canto_dir = os.path.abspath(os.getcwd()) |
|||
canto_files = os.listdir(canto_dir) |
|||
|
|||
def rename_canto(canto): |
|||
""" |
|||
Take txt canto files from `dante-visualized` and change the name styling |
|||
""" |
|||
# parse canto name |
|||
# put it back together |
|||
print('canto name: ' + canto) |
|||
new_canto = re.sub('\_','-',canto) |
|||
new_canto = re.sub('Canto[IVX]*-','',new_canto) |
|||
new_canto = new_canto.lower() |
|||
print('--> new canto name: ' + new_canto) |
|||
os.rename(canto,new_canto) |
|||
|
|||
for canto in canto_files: |
|||
if canto != 'rename-txt.py' and canto != '.rename-txt.py.swp': |
|||
rename_canto(canto) |
|||
@ -0,0 +1,124 @@ |
|||
import re |
|||
import os |
|||
import xml.etree.ElementTree as ET |
|||
|
|||
""" |
|||
This script takes xml canto files and counts the number of unique words as |
|||
defined by the value of `<w>` nodes. |
|||
|
|||
8 files are generated: csv and html of |
|||
- sorted numerically and alphabetically |
|||
- sorted numerically and alphabetically with blacklist applied |
|||
""" |
|||
|
|||
cantoDir = os.path.abspath(os.getcwd()) |
|||
|
|||
blacklist = ('e', 'che', 'la', 'di', 'a', 'per', '’l', 'mi', 'tu', 'con', |
|||
'lo', 'sì', 'sua', 'li', 'del', 'si', 'nel', 'era', 'ch’i’', 'è', |
|||
'là', 'le', 'de', 'quella', 'poi', 'ma', 'il', 'già', '’n', 'una', |
|||
'tanto', 'se’', 'quel', 'qual', 'perché', 'in', 'come', 'al', 'da', |
|||
'un', 'i', 'ti', 'a', 'per', 'con', 'lo', 'al', 'poi', 'E', 'il') |
|||
|
|||
def createWordCountDict(cantoFile): |
|||
""" |
|||
for node in xpathQueryAllWordNodes: |
|||
if xpathValue(node) in wordCountDict(keys): |
|||
wordCountDict[node] += 1 |
|||
else: |
|||
wordCountDict[xpathValue(node)] = 1 |
|||
""" |
|||
wordCountDict = dict() |
|||
tree = ET.parse(cantoFile) |
|||
root = tree.getroot() |
|||
# TODO |
|||
words = root.findall(".//{http://www.tei-c.org/ns/1.0}w") |
|||
for word in words: |
|||
if word.text in wordCountDict: |
|||
wordCountDict[word.text] += 1 |
|||
else: |
|||
wordCountDict[word.text] = 1 |
|||
return wordCountDict |
|||
|
|||
def numSort(wordCountDict): |
|||
# Need reverse = True to sort greatest to least |
|||
wordCountDictNumSorted = dict(sorted(wordCountDict.items(), key=lambda item: item[1], reverse = True)) |
|||
return wordCountDictNumSorted |
|||
|
|||
def alphaSort(wordCountDict): |
|||
# Capital letters come first with this implementation |
|||
wordCountDictAlphaSorted = dict(sorted(wordCountDict.items())) |
|||
return wordCountDictAlphaSorted |
|||
|
|||
def blackSort(wordCountDictSorted): |
|||
sortedCopy = dict(wordCountDictSorted) |
|||
for word in wordCountDictSorted: |
|||
if word in blacklist: |
|||
del sortedCopy[word] |
|||
return sortedCopy |
|||
|
|||
def dictToCsv(sortedDict, sortType, cantoFile): |
|||
""" |
|||
write a sorted dict to a CSV file for that canto |
|||
name it based on sorting type (numerical or alphabetical) |
|||
abbreviate sortType as `num` or `alpha` |
|||
""" |
|||
csvCanto = cantoFile.split('.')[0] + '-' + sortType + '.csv' |
|||
cantoNum = str(int(cantoFile.split('-')[0])) |
|||
canticle = cantoFile.split('-')[1] |
|||
|
|||
with open(csvCanto, 'w') as csvCanto: |
|||
cantoInformation = canticle + ',' + cantoNum + '\n' |
|||
csvCanto.write(cantoInformation) |
|||
for word, count in sortedDict.items(): |
|||
csvInput = word + ',' + str(count) + '\n' |
|||
csvCanto.write(csvInput) |
|||
csvCanto.close() |
|||
|
|||
def dictToHtml(sortedDict, sortType, cantoFile): |
|||
htmlCanto = cantoFile.split('.')[0] + '-' + sortType + '.html' |
|||
with open(htmlCanto, 'w') as htmlCanto: |
|||
cantoNum = str(int(cantoFile.split('-')[0])) |
|||
canticle = cantoFile.split('-')[1].title() |
|||
if sortType == 'num': |
|||
sortInformation = 'Words sorted by most to least occurrences.' |
|||
elif sortType == 'num-black': |
|||
sortInformation = 'Words sorted by most to least occurrences, with blacklist applied.' |
|||
elif sortType == 'alpha': |
|||
sortInformation = 'Words sorted alphabetically.' |
|||
elif sortType == 'alpha-black': |
|||
sortInformation = 'Words sorted alphabetically, with blacklist applied.' |
|||
cantoHeaders = '<h1>' + canticle + ' ' + cantoNum + '</h1>\n<h3>' + sortInformation + '</h3>\n' |
|||
boilerplate = '<html>\n' + cantoHeaders + '<table>\n<tr>\n<th>Word</th><th>Count</th>\n</tr>\n' |
|||
boilerclose = '</table>\n</html>' |
|||
htmlCanto.write(boilerplate) |
|||
for word, count in sortedDict.items(): |
|||
htmlInput = '<tr>\n<td>' + word + '</td><td>' + str(count) + '</td>\n</tr>\n' |
|||
htmlCanto.write(htmlInput) |
|||
htmlCanto.write(boilerclose) |
|||
htmlCanto.close() |
|||
|
|||
def main(): |
|||
for canto in os.listdir(cantoDir): |
|||
if re.search('\.xml$', canto): |
|||
cantoFile = canto |
|||
wordCountDict = createWordCountDict(cantoFile) |
|||
|
|||
# create sorted dicts |
|||
wordCountDictNumSorted = numSort(wordCountDict) |
|||
wordCountDictNumSortedBlack = blackSort(wordCountDictNumSorted) |
|||
wordCountDictAlphaSorted = alphaSort(wordCountDict) |
|||
wordCountDictAlphaSortedBlack = blackSort(wordCountDictAlphaSorted) |
|||
|
|||
# create output files |
|||
dictToCsv(wordCountDictNumSorted, 'num', cantoFile) # `num-full`: sort numerically with no blacklist |
|||
dictToCsv(wordCountDictNumSortedBlack, 'num-black', cantoFile) # `num`: sort numerically with blacklist |
|||
dictToCsv(wordCountDictAlphaSorted, 'alpha', cantoFile) |
|||
dictToCsv(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) |
|||
|
|||
dictToHtml(wordCountDictNumSorted, 'num', cantoFile) |
|||
dictToHtml(wordCountDictNumSortedBlack, 'num-black', cantoFile) |
|||
dictToHtml(wordCountDictAlphaSorted, 'alpha', cantoFile) |
|||
dictToHtml(wordCountDictAlphaSortedBlack, 'alpha-black', cantoFile) |
|||
|
|||
|
|||
main() |
|||
Loading…
Reference in new issue