root/hodgestar/PythonCode/SmallScripts/fetch-canting.py

#!env python

"""
Module to fetch the 1736 Canting Dictionary from:
http://www.fromoldbooks.org/NathanBailey-CantingDictionary/transcription.html
"""

import HTMLParser
import urllib2
import urllib

class LetterPageParser(HTMLParser.HTMLParser, object):
    def __init__(self, sBaseUrl):
        self._aURLs = []
        self._bInTd = False
        self._sBaseUrl = sBaseUrl
        super(LetterPageParser, self).__init__()

    def get_urls(self):
        """Return the URLs"""
        return list(self._aURLs)

    def reset(self):
        """Reset the parser"""
        super(LetterPageParser, self).reset()
        self._aURLs = []
        self._bInTd = False

    def handle_starttag(self, sTag, aAttr):
        if not self._bInTd:
            if sTag.lower() == "td":
                self._bInTd = True
        else:
            if sTag.lower() == "a":
                dAttr = dict(aAttr)
                self._aURLs.append(
                    urllib.basejoin(self._sBaseUrl, dAttr["href"])
                )

    def handle_endtag(self, sTag):
        if self._bInTd:
            if sTag.lower() == "td":
                self._bInTd = False

    def handle_data(self, sData): pass
    def handle_charref(self, sName): pass
    def handle_entityref(self, sName): pass


class WordPageParser(HTMLParser.HTMLParser, object):
    def __init__(self):
        self._sWord = None
        self._sDef = None
        self._sData = ""
        self._bWordP = False
        super(WordPageParser, self).__init__()

    def get_word(self):
        """Return the URLs"""
        return self._sWord, self._sDef

    def reset(self):
        """Reset the parser"""
        self._sWord = None
        self._sDef = None
        self._sData = ""
        super(WordPageParser, self).reset()

    def handle_starttag(self, sTag, aAttr):
        if sTag.lower() == "p":
            self._sData = ""
            self._bWordP = False
        elif sTag.lower() == "span":
            dAttr = dict(aAttr)
            if dAttr.get("class") == "headword":
                self._bWordP = True

    def handle_endtag(self, sTag):
        if self._bWordP:
            if sTag.lower() == "p":
                self._sDef = self._sData
            elif sTag.lower() == "span":
                self._sWord = self._sData

    def handle_data(self, sData):
        self._sData += sData

    def handle_charref(self, sName): pass
    def handle_entityref(self, sName): pass


def fetch_word(sUrl):
    """Fetch the word from sUrl, returning the word and definition."""
    oP = WordPageParser()
    f = urllib2.urlopen(sUrl)
    try:
        oP.feed(f.read())
    finally:
        f.close()
    return oP.get_word()

def fetch_letter(sUrl):
    """Fetch all the words from the letter page sUrl."""
    oP = LetterPageParser(sUrl)
    f = urllib2.urlopen(sUrl)
    try:
        oP.feed(f.read())
    finally:
        f.close()

    for sWordUrl in oP.get_urls():
        yield fetch_word(sWordUrl)

def fetch_all(sUrl):
    """Fetch all the words using sUrl as the base url."""
    for sL in list("ABCDEFGH") + ["IJ"] + \
              list("KLMNOPQRST") + ["UV"] + \
              list("XYZ"):
        for tWordDef in fetch_letter("%s/%s/" % (sUrl, sL)):
            yield tWordDef

if __name__ == "__main__":
    import sys

    sBaseUrl = "http://www.fromoldbooks.org/NathanBailey-CantingDictionary/"

    for sWord, sDef in fetch_all(sBaseUrl):
        sys.stdout.write(sWord)
        sys.stdout.write("\n    ")
        sys.stdout.write(sDef)
        sys.stdout.write("\n\n")
        sys.stdout.flush()