#!/usr/bin/env python
"""
sitedb.py - Create a searchable site database
License: GPL 2; share and enjoy!
Author: Sean B. Palmer, inamidst.com

Usage: 
   ./sitedb.py <rootpath> <dbpath>  - create database at dbpath
   ./sitedb.py --help               - help and usage notes

Bugs: 
   * Duplicates entries for words appearing more than once in a line
   * The robots.txt parsing doesn't appear to be working properly
"""

import sys, re, os, dbhash
from robotparser import RobotFileParser

r_word = re.compile(r"(?<!')[A-Za-z](?:[A-Za-z-]*[A-Za-z])?")
indexables = ('.html', '.txt', '.cgi', '.py', '.c', '.css', '.php')

class RobotFileWrapper(object): 
   def __init__(self, rp): 
      self.rp = rp
   def readable(self, path): 
      if not self.rp: 
         return True
      return self.rp.can_fetch('*', path)

def mkrparser(rootpath): 
   """Make a robots.txt parser for the site."""
   robotstxt = os.path.join(rootpath, 'robots.txt')
   if os.path.exists(robotstxt):
      rp = RobotFileParser()
      rp.set_url(robotstxt)
      rp.read() # @@ this reads borked files too
   else: rp = None
   return RobotFileWrapper(rp)

def index(fn, db): 
   """Index each line of fn in db."""
   try: f = open(fn)
   except: return False
   linecount = 1
   for line in f: # Efficient iteration
      lineid = fn + '\t' + str(linecount)
      db[lineid] = line

      for word in r_word.findall(line): 
         if db.has_key(word): 
            db[word] += '\n' + lineid
         else: db[word] = lineid
      linecount += 1
   f.close()

def makedb(rootpath, dbpath): 
   """Make a site db, without overwriting any previous database."""
   if os.path.exists(dbpath): 
      print >> sys.stderr, "Won't overwrite %s" % dbpath
      sys.exit(1)

   rp = mkrparser(rootpath)
   db = dbhash.open(dbpath, 'n')

   def indexable(fn): 
      for ext in indexables: 
         if fn.endswith(ext): return True
      return False

   indexed = 0
   print "Indexing files... this may take a while!"
   for (dirpath, dirnames, filenames) in os.walk(rootpath): 
      for name in filenames: 
         fn = os.path.join(dirpath, name)
         if rp.readable(fn): 
            if indexable(fn): 
               index(fn, db)
               sys.stderr.write('.')
               indexed += 1
   db.close()

   print "Success! Indexed %s files." % indexed
   return True

def findword(dbpath, word, lines=True): 
   result = []
   db = dbhash.open(dbpath, 'r')
   if db.has_key(word): 
      for lineid in db[word].split('\n'): 
         if not lines: result.append(lineid)
         else: result.append((lineid, db[lineid]))
   db.close()
   return result

r_prent = re.compile(r'&(?!amp;)')
def condquot(text): 
   text = text.replace('<', '&lt;')
   return r_prent.sub('&amp;', text)

def mklinks(html, query):
   def replacement(m):
      word = m.group(0)
      return '<a href="%s">%s</a>' % (query + word, word)
   return r_word.sub(replacement, html)

def fwhtml(dbpath, word, strip=None, query=None): 
   result = ['<ul>']
   template = '<li><a href="%s" class="uri">%s:%s</a> %s</li>'
   for (lineid, line) in findword(dbpath, word): 
      fn, n = lineid.split('\t')
      if strip and fn.startswith(strip[0]): 
         fn = strip[1] + fn[len(strip[0]):]
      line = condquot(line)
      if query: line = mklinks(line, query)
      result.append(template % (fn, fn, n, line))
   result.append('</ul>')
   return '\n'.join(result)

def main(args=None): 
   if args is None: 
      args = sys.argv[1:]

   if len(args) == 2: 
      rootpath, dbpath = tuple(args)
      makedb(rootpath, dbpath)
   else: print __doc__.lstrip()

if __name__=="__main__": 
   main()