root/hodgestar/PythonCode/Duophage/duophage.py

#!/usr/bin/python

try:
    from HTMLParser import HTMLParser
    bHaveHTMLParser = True
except ImportError:
    HTMLParser = object
    bHaveHTMLParser = False

try:
    from libxml2 import SAXCallback, createPushParser
    bHaveLibxml2 = True
except:
    SAXCallback = object
    bHaveLibxml2 = False

import urllib, sys, base64
import logging

class DuoParserBase(object):
    """Base class with helper methods for Duophage parsers.

       Sub-classes must provide:
         self._sBaseUrl : base URL for the document.
         self._fOut : output file.
       """

    def _urlToAbsUrl(self,sUrl):
        """
        Convert a relative or absolute URL to an absolute URL.
        """
        return urllib.basejoin(self._sBaseUrl,sUrl)

    def _urlToData(self,sUrl):
        """
        Convert any URL to an equivalent data: URL.
        """
        if sUrl.startswith('data:'): return sUrl

        sUrl = self._urlToAbsUrl(sUrl)
        fSrc = urllib.urlopen(sUrl)
        sData = base64.standard_b64encode(fSrc.read())
        oInfo = fSrc.info()
        sMime = oInfo['Content-Type']

        return 'data:' + sMime + ';base64,' + sData

    def _handleStartTag(self,sTag,aAttrs,bEnd=False):
        """
        Handle a start tag.

        If bEnd is True, this is also an end tag.
        """
        if aAttrs:
            dAttrs = dict(aAttrs)
        else:
            dAttrs = {}

        # Set a new base url if we encounter a base tag
        if sTag == 'base' and dAttrs.has_key('href'):
            self._sBaseUrl = dAttrs['href']

        # Canonicalise non-data URLs
        if dAttrs.has_key('href') and not dAttrs['href'].startswith('data:'):
            dAttrs['href'] = self._urlToAbsUrl(dAttrs['href'])

        # Convert src URLs to data
        if dAttrs.has_key('src'):
            dAttrs['src'] = self._urlToData(dAttrs['src'])

        # Convert links to stylesheets to data            
        if sTag == 'link' and dAttrs.get('rel','').lower() == 'stylesheet':
            dAttrs['href'] = self._urlToData(dAttrs['href'])

        # write out new tag            
        self._fOut.write("<" + sTag)
        for k,v in dAttrs.iteritems():
            if v is None:
                self._fOut.write(" " + k + "='" + k + "'")
            else:
                self._fOut.write(" " + k + "='" + v + "'")
        if bEnd:
            self._fOut.write("/>")
        else:
            self._fOut.write(">")


class DuoParser(HTMLParser, DuoParserBase):
    """
    Duophage parser based on standard Python HTMLParser.
    """
    def __init__(self, fOut, sBaseUrl):
        super(DuoParser, self).__init__()
        self._fOut = fOut
        self._sBaseUrl = sBaseUrl

    def handle_starttag(self, sTag, aAttrs): self._handleStartTag(sTag, aAttrs, False)
    def handle_startendtag(self, sTag, aAttrs): self._handleStartTag(sTag, aAttrs, True)
    def handle_endtag(self, sTag): self._fOut.write("</" + sTag + ">")
    def handle_data(self, sData): self._fOut.write(sData)
    def handle_charref(self, sName): self._fOut.write("&" + sName + ";")
    def handle_entityref(self, sName): self._fOut.write("&" + sName + ";")
    def handle_comment(self, sData): self._fOut.write("<!--" + sData + "-->")
    def handle_decl(self, sDecl): self._fOut.write("<!" + sDecl + ">")
    def handle_pi(self, sData): self._fOut.write("<?" + sData + ">")


class DuoLxmlParser(SAXCallback, DuoParserBase):
    """
    Duophage parser for libxml2.
    """
    def __init__(self,fOut,sBaseUrl):
        super(DuoLxmlParser, self).__init__()
        self._fOut = fOut
        self._sBaseUrl = sBaseUrl

    def startElement(self, sTag, aAttrs): self._handleStartTag(sTag, aAttrs, False)
    def endElement(self, sTag): self._fOut.write("</" + sTag + ">")
    def characters(self, sData): self._fOut.write(sData)

    # def handle_charref(self, sName): self._fOut.write("&" + sName + ";")

    def reference(self, sName): self._fOut.write("&" + sName + ";")
    def comment(self, sData): self._fOut.write("<!--" + sData + "-->")

    # def handle_decl(self, sDecl): self._fOut.write("<!" + sDecl + ">")
    # def handle_pi(self, sData): self._fOut.write("<?" + sData + ">")

    def error(self, msg): logging.warn(msg)

    # Other methods which might be worth implementing:
    #
    #  attributeDecl(self, elem, name, type, defi, defaultValue, nameList)
    #      called when an ATTRIBUTE definition has been found
    #
    #  cdataBlock(self, data)
    #      called when CDATA section have been read, data is the string
    #      containing the data, multiple consecutive cdataBlock() callback
    #      are possible.
    #
    #  elementDecl(self, name, type, content)
    #      called when an ELEMENT definition has been found
    #
    #  entityDecl(self, name, publicId, systemID, notationName)
    #      called when an unparsed ENTITY declaration has been found,
    #      name is the entity name and publicId,, systemID are the entity
    #      public and system identifier for that entity if available,
    #      and notationName indicate the associated NOTATION
    #
    #  externalSubset(self, name, externalID, systemID)
    #      called when a DOCTYPE declaration has been found, name is the
    #      DTD name and externalID, systemID are the DTD public and system
    #      identifier for that DTd if available
    #
    #  ignorableWhitespace(self, data)
    #      called when potentially ignorable white spaces have been found
    #  
    # internalSubset(self, name, externalID, systemID)
    #      called when a DOCTYPE declaration has been found, name is the
    #      DTD name and externalID, systemID are the DTD public and system
    #      identifier for that DTD if available
    #  
    #  notationDecl(self, name, externalID, systemID)
    #      called when an NOTATION declaration has been found, name is the
    #      notation name and externalID, systemID are the notation public and
    #      system identifier for that notation if available
    #  
    #  processingInstruction(self, target, data)
    #      called when a PI has been found, target contains the PI name and
    #      data is the associated data in the PI
    #
    #  error(self, msg)

def main(aArgs):
    if len(aArgs) != 2:
        print aArgs[0], "'http://example/foo'", "> output.html"
        print "  ---"
        print "  Duophage makes HTML pages self-contained by looking for"
        print "  src='...' attributes, downloading the data and encoding"
        print "  it into a 'data:...' URI."
        return 1

    sUrl = aArgs[1]

    # Current the libxml2 SAX parser is looking even less robust
    # against bad HTML than the stock HTMLParser :/
    #   if bHaveLibxml2:
    #        oH = DuoLxmlParser(sys.stdout, sUrl)
    #        oP = createPushParser(oH, "", 0, sUrl)
    #        for s in urllib.urlopen(sUrl):
    #            oP.parseChunk(s, len(s), 0)
    #        oP.parseChunk("", 0, 1)

    if bHaveHTMLParser:
        oP = DuoParser(sys.stdout, sUrl)
        for s in urllib.urlopen(sUrl):
            oP.feed(s)

    else:
        print "No parsers found (tried HTMLParser and libxml2)."
        return 2

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))