#!/usr/bin/env python
"""
words.py - Extract All Words from Wikipedia...
Author: Sean B. Palmer, inamidst.com
"""

import sys, re, bz2

r_word = re.compile(r"(?<!')\b[A-Za-z](?:[A-Za-z'-]*[A-Za-z])?\b")

def main(): 
   vocabulary = set()
   name = 'enwiki-20060208-pages-articles.xml.bz2'

   wikipedia = bz2.BZ2File(name, 'r', 2048)
   for i, line in enumerate(wikipedia): 
      words = (word for word in r_word.findall(line) 
               if not word.count("--") and 
                  not word.count("''"))
      vocabulary.update(words)

      if not (i % 500000): 
         print >> sys.stderr, i
   wikipedia.close()

   for word in vocabulary: 
      print word

if __name__=="__main__": 
   main()