#!/usr/bin/env python
"""
xhtmlnorm.py - XHTML Normaliser
Author: Sean B. Palmer, inamidst.com
Options: 
* Entities -> numeric (-e numeric), utf-8 (-e utf-8), remove (-e remove)
* CDATA -> entities (-t entities), remove (-t remove)
* Doctype -> remove (-d)
* Comments -> remove (-c)
* PI -> remove (-p)
* Encoding/XMLDecl -> ?
"""

import sys, re
from htmlentitydefs import name2codepoint
from optparse import OptionParser

r_cdata = re.compile(r'<!\[CDATA\[((?:[^\]]+|\](?!\]>))*)\]\]>')
r_comment = re.compile(r'<!--((?:[^-]|(?:-[^-]))*)-->')
r_delimiter = re.compile(r'<|&')
r_doctype = re.compile(r'(?m)(<!DOCTYPE\s+\S+[^\[]+?(\[[^\]]+?\])?\s*>)')
r_entity = re.compile(r'&[A-Za-z0-9#]+;')
r_pi = re.compile(r'<\?(\S+)[\t\n\r ]+(([^\?]+|\?(?!>))*)\?>')
# r_tag = re.compile(r'(?ms)<[^>]+>')
r_tag = re.compile(r'<([^\'">]+("[^"]+"|\'[^\']+\')?)*>')

def escape(data): 
   data = data.replace('&', '&amp;')
   data = data.replace('<', '&lt;')
   return data.replace('>', '&gt;')

class HTMLNormaliser(object): 
   def __init__(self, options): 
      self.options = options

   def inputFilename(self, filename): 
      if filename in ('-', '/dev/stdin'): 
         self.input = sys.stdin
      else: self.input = open(filename)

   def outputFilename(self, filename): 
      if filename in ('-', '/dev/stdout'): 
         self.output = sys.stdout
      else: self.output = open(filename, 'w')

   def closeInput(self): 
      if self.input is not sys.stdin: 
         self.input.close()

   def closeOutput(self): 
      if self.output is not sys.stdout: 
         self.output.close()

   def normalise(self): 
      self.bytes = self.input.read()

      while True: 
         m = r_delimiter.search(self.bytes)
         if not m: 
            self.output.write(self.bytes)
            break

         pos = m.end() - 1
         self.output.write(self.bytes[:pos])
         self.bytes = self.bytes[pos:]

         if self.bytes.startswith('&'): 
            self.parseEntity()
         elif self.bytes.startswith('<![CDATA['): 
            self.parseCDATA()
         elif self.bytes.startswith('<!DOCTYPE'): 
            self.parseDOCTYPE()
         elif self.bytes.startswith('<!--'): 
            self.parseComment()
         elif self.bytes.startswith('<?'): 
            self.parsePI()
         else: self.parseTag()

         # superelse: raise ValueError('Weird: ' + bytes[:10] + '...')

   def parseEntity(self): 
      m = r_entity.match(self.bytes)
      try: entity = m.group(0)
      except: raise Exception(self.bytes[:25] + '...')
      self.bytes = self.bytes[m.end():]

      if self.options.entities == 'numeric': 
         # @@ But preserve lt/gt/amp/apos?
         if entity.startswith('&#x'): 
            self.output.write('&#x%04X;' % int(entity[3:-1], 16))
         elif entity.startswith('&#'): 
            self.output.write('&#x%04X;' % int(entity[2:-1]))
         else: 
            cp = name2codepoint[entity[1:-1]]
            self.output.write('&#x%04X;' % cp)

      elif self.options.entities == 'utf-8': 
         if entity.startswith('&#x'): 
            cp = int(entity[3:-1], 16)
            self.output.write(unichr(cp).encode('utf-8'))
         elif entity.startswith('&#'): 
            cp = int(entity[2:-1])
            self.output.write(unichr(cp).encode('utf-8'))
         else: 
            char = name2codepoint[entity[1:-1]]
            self.output.write(unichr(char).encode('utf-8'))

      elif self.options.entities != 'remove': 
         self.output.write(entity)

   def parseCDATA(self): 
      m = r_cdata.match(self.bytes)
      cdata = m.group(0)
      self.bytes = self.bytes[m.end():]
      if self.options.whitespace: 
         self.bytes = self.bytes.lstrip('\t\r\n ')

      if self.options.cdata == 'entities': 
         self.output.write(escape(m.group(1)))
      elif self.options.cdata != 'remove': 
         self.output.write(cdata)

   def parseDOCTYPE(self): 
      m = r_doctype.match(self.bytes)
      doctype = m.group(0)
      self.bytes = self.bytes[m.end():]
      if self.options.whitespace: 
         self.bytes = self.bytes.lstrip('\t\r\n ')

      if not self.options.doctype: 
         self.output.write(doctype)

   def parseComment(self): 
      m = r_comment.match(self.bytes)
      comment = m.group(0)
      self.bytes = self.bytes[m.end():]
      if self.options.whitespace: 
         self.bytes = self.bytes.lstrip('\t\r\n ')

      if not self.options.comments: 
         self.output.write(comment)

   def parsePI(self): 
      m = r_pi.match(self.bytes)
      pi = m.group(0)
      self.bytes = self.bytes[m.end():]
      if self.options.whitespace: 
         self.bytes = self.bytes.lstrip('\t\r\n ')

      if not self.options.pi: 
         self.output.write(pi)

   def parseTag(self): 
      m = r_tag.match(self.bytes)
      tag = m.group(0)
      self.bytes = self.bytes[m.end():]

      self.output.write(tag)

def normalise(input, output, options): 
   n = HTMLNormaliser(options)
   n.inputFilename(input)
   n.outputFilename(output)
   n.normalise()

def test(): 
   import textwrap
   html = textwrap.dedent("""\
      <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" >
      <html xmlns="http://www.w3.org/1999/xhtml">
      <head><title>Test Document</title></head>
      <body>
      <!-- This is a comment -->
      <p title="test>">Caf&eacute;, and&#x2014;test&#x203D;.
      <?test Processing Instruction?></p>
      <pre><![CDATA[
      <p>Test CDATA <em>section</em>.</p>
      ]]>
      </pre>
      </body>
      </html>
   """)

   reference = textwrap.dedent("""\
      <html xmlns="http://www.w3.org/1999/xhtml">
      <head><title>Test Document</title></head>
      <body>
      <p title="test>">Caf&#x00E9;, and&#x2014;test&#x203D;.
      </p>
      <pre>
      &lt;p&gt;Test CDATA &lt;em&gt;section&lt;/em&gt;.&lt;/p&gt;
      </pre>
      </body>
      </html>
   """)

   from cStringIO import StringIO
   input = StringIO()
   input.write(html)
   input.seek(0)

   output = StringIO()

   options = type('_', (object,), {})()

   options.entities = 'numeric'
   options.cdata = 'entities'

   options.doctype = True
   options.comments = True
   options.pi = True
   options.whitespace = True

   n = HTMLNormaliser(options)
   n.input = input
   n.output = output
   n.normalise()

   output.seek(0)
   assert output.read() == reference
   print reference
   print "Test passed"

def opt(parser, flags, help): 
   short, long = flags[1:].split('/', 1)
   if '=' in long: 
      long, metavar = long.split('=', 1)
      kargs = dict(default=False, help=help, metavar=metavar)
   else: kargs = dict(action='store_true', default=False, help=help)
   parser.add_option('-' + short, '--' + long, **kargs)

def main(): 
   parser = OptionParser(usage="%prog [options] <input>")

   # Input and Output
   opt(parser, "-i/inplace", "modify file in-place")
   opt(parser, "-o/output=FN", "output filename")

   # Normalisation options
   opt(parser, "-e/entities=OP", "entity ops (numeric/utf-8/remove)")
   opt(parser, "-t/cdata=OP", "cdata ops (entities/remove)")
   opt(parser, "-d/doctype", "remove doctype")
   opt(parser, "-c/comments", "remove comments")
   opt(parser, "-p/pi", "remove processing instructions")
   opt(parser, "-w/whitespace", "remove whitespace")
   opt(parser, "-a/all", "normalise all")

   options, args = parser.parse_args()

   if options.inplace and options.output: 
      parser.error("-i and -o are mutually exclusive")

   if options.all: 
      options.entities = 'numeric'
      options.cdata = 'entities'

      options.doctype = True
      options.comments = True
      options.pi = True
      options.whitespace = True

   if len(args) > 1: 
      parser.error("Maximum of one input files")

   if args: 
      input = args[0]
   else: input = '-'

   if options.output: 
      output = options.output
   elif options.inplace: 
      if input in ('-', '/dev/stdin'): 
         parser.error("Can't use in-place with stdin")
      output = input
   else: output = '-'

   normalise(input, output, options)

if __name__=="__main__": 
   main()