#!/usr/bin/env python
import sys, re
lexpats = [
('C', 'c[jtp]?he*'),
('A', 'ai*[nm]?'),
('T', '[kt]e*'),
('P', '[pf]'),
('O', 'o[ei]*[nm]?'),
('Y', 'y'),
('D', 'de*'),
('L', '[lr]e*'),
('S', 'se*'),
('X', '.')
]
patterns = '|'.join('(%s)' % lexpat[1] for lexpat in lexpats)
names = [lexpat[0] for lexpat in lexpats]
r_CatP = re.compile(patterns)
def convert(text):
result = []
while text:
m = r_CatP.match(text)
if m:
C, A, T, P, O, Y, D, L, S, X = m.groups()
if C: result.append('C')
if A: result.append('A')
if T: result.append('T')
if P: result.append('P')
if O: result.append('O')
if Y: result.append('Y')
if D: result.append('D')
if L: result.append('L')
if S: result.append('S')
if X: result.append('X')
text = text[m.end():]
else:
result.append(text[0])
text = text[1:]
return ''.join(result)
docs = {'eva2-v101.txt': {}, 'eva2-stolfi.txt': {}}
for doc in ('eva2-v101.txt', 'eva2-stolfi.txt'):
f = open(doc)
prev = None
for line in f:
page, text = line.strip().split(' ', 1)
if len(page) == 3:
page = 'f00' + page[1:]
elif len(page) == 4 or (len(page) == 5 and page[-1].isdigit()):
page = 'f0' + page[1:]
if page != prev:
docs[doc][page] = []
prev = page
docs[doc][page].append(text)
f.close()
v101_pages = docs['eva2-v101.txt'].keys()
stolfi_pages = docs['eva2-stolfi.txt'].keys()
pages = set(v101_pages + stolfi_pages)
lengths = []
for page in docs['eva2-v101.txt']:
p = []
for line in docs['eva2-v101.txt'][page]:
line = convert(line.strip().replace(' ', ''))
lengths.append(len(line))
p.append(line)
docs['eva2-v101.txt'][page] = p
average = sum(lengths) / len(lengths)
# average line length is 38
# http://en.wikibooks.org/
# /wiki/Algorithm_Implementation/Strings/Levenshtein_distance#Python
def levenshtein(s1, s2):
if len(s1) < len(s2):
return levenshtein(s2, s1)
if not s1:
return len(s2)
previous_row = xrange(len(s2) + 1)
for i, c1 in enumerate(s1):
current_row = [i + 1]
for j, c2 in enumerate(s2):
insertions = previous_row[j + 1] + 1
deletions = current_row[j] + 1
substitutions = previous_row[j] + (c1 != c2)
current_row.append(min(insertions, deletions, substitutions))
previous_row = current_row
return previous_row[-1]
def pairs(items):
length = len(items)
end = length - 1
for i in xrange(length):
for j in xrange(i + 1, length):
yield items[i], items[j]
def frequencies(items):
result = {}
for item in items:
if item == '':
item = '|'
try: result[item] += 1
except KeyError: result[item] = 1
# pairs = [(a, b) for (b, a) in result.iteritems()]
return sorted(result.items()) # , reverse=True)
print """\
Voynich Distances
"""
# for page in sorted(docs['eva2-v101.txt'].keys()):
# print page
# print
# for line in docs['eva2-v101.txt'][page]:
# print line
# print
for page in sorted(docs['eva2-v101.txt'].keys()):
# Remove lines shorter than half the average length
for line in docs['eva2-v101.txt'][page][:]:
if len(line) < (average / 2):
docs['eva2-v101.txt'][page].remove(line)
print '
%s
' % page
distances = []
for a, b in pairs(docs['eva2-v101.txt'][page]):
distance = levenshtein(a, b)
distances.append(distance)
print '