#!/usr/bin/python
"""Extract information about RSS subscribership from hosted RSS readers
by parsing their User-Agent data."""

__author__ = "Nelson Minar <nelson@monkey.org>"
__url__ = "http://www.nelson.monkey.org/~nelson/weblog/"

# I place this code in the public domain. Use it for what you want.
# No warranty or guarantee, of course.

import re, unittest, sys, gzip

# Regexps for each aggregator
# Regexps capture data in named groups: readers, and views
regexps = (
    ('Yahoo', r'''
       YahooFeedSeeker.*;
       \s*users\s*(?P<readers>\d+);
       \s*views\s*(?P<views>\d+)
     '''),
    ('LiveJournal', r'''
       LiveJournal\.com.*;
       \s*(?P<readers>\d+)\s*readers
     '''),
    ('NewsIsFree', r'''
       NIF.*
       users:(?P<readers>\d+)
     '''),
    ('BlogLines', r'''
       Bloglines.*
       ;\s*(?P<readers>\d+)\s*subscribers
     '''),
    ('NewsGator', r'''
       NewsGatorOnline.*
       ;\s*(?P<readers>\d+)\s*subscribers
     '''),
)

# Basic regexp prepended to list above, used for all user agents.
baseRE = r'''
  \[
  (?P<date>\d+/\S+/\d\d\d\d)
  :
  (?P<time>\d\d:\d\d:\d\d)
  \s*
  (?P<tz>[^\]]+)
  \]\s+"
  GET\s(?P<url>\S+).*
'''

# Compiled regexps; these are used in the matcher.
compiledRegexps = []
for name, pattern in regexps:
    compiledRegexps.append((name, re.compile(baseRE + pattern, re.VERBOSE)))


def matchLine(line):
    """Try finding a match against the given log line.
    Return name, match object if found, else None, None"""
    for name, r in compiledRegexps:
        m = r.search(line)
        if m:
            return name, m
    return None, None

def dumpMatches(filename):
    "Simple demo of matchLines. Dumps all matches from the supplied file."
    if f.endswith("gz"):
        fp = gzip.open(filename)
    else:
        fp = file(filename)
    for line in fp:
        name, m = matchLine(line)
        if name:
            print "%s\t%s\t%6s\t%s" % \
                  (m.group('date'), name, m.group('readers'), m.group('url'))
    fp.close()

class RegexpTests(unittest.TestCase):
    "Some tests for the regexp matcher"
    def testMatchLineNone(self):
        self.assertEqual((None, None), matchLine("foo"))

    def testYahoo(self):
        name, m = matchLine('self.assertEqual(None, oc2.my.scd.yahoo.net - - [26/Mar/2005:04:18:10 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.0" 304 - "-" "YahooFeedSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://my.yahoo.com/s/publishers.html; users 2; views 35)" 1 www.nelson.monkey.org 0')
        self.assertEqual('Yahoo', name)
        self.assertEqual('2', m.group('readers'))
        self.assertEqual('35', m.group('views'))

    def testLiveJournal(self):
        name, m = matchLine('livejournal.com - - [20/Mar/2005:07:20:30 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.1" 304 - "-" "LiveJournal.com (webmaster@livejournal.com; for http://www.livejournal.com/users/nelsonblog/; 2 readers)" 1 www.nelson.monkey.org 0')
        self.assertEqual('LiveJournal', name)
        self.assertEqual('2', m.group('readers'))

    def testNewsIsFree(self):
        name, m = matchLine('spider.newsisfree.com - - [20/Mar/2005:10:01:22 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.0" 200 19288 "-" "NIF/1.1  (http://www.newsisfree.com/robot.php  users:0)" 6 www.nelson.monkey.org 0')
        self.assertEqual('NewsIsFree', name)
        self.assertEqual('0', m.group('readers'))

    def testBlogLines(self):
        name, m = matchLine('crawler01.bloglines.com - - [20/Mar/2005:07:00:49 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.1" 200 5855 "-" "Bloglines/2.0 (http://www.bloglines.com; 288 subscribers)" 0 www.nelson.monkey.org 70')
        self.assertEqual('BlogLines', name)
        self.assertEqual('288', m.group('readers'))

    def testNewsGator(self):
        name, m = matchLine('64.78.155.100 - - [20/Mar/2005:07:01:57 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.1" 304 - "-" "NewsGatorOnline/2.0 (http://www.newsgator.com; 31 subscribers)" 0 www.nelson.monkey.org 0')
        self.assertEqual('NewsGator', name)
        self.assertEqual('31', m.group('readers'))

    def testBaseRE(self):
        name, m = matchLine('crawler01.bloglines.com - - [20/Mar/2005:07:00:49 -0800] "GET /~nelson/weblog/index.rss091 HTTP/1.1" 200 5855 "-" "Bloglines/2.0 (http://www.bloglines.com; 288 subscribers)" 0 www.nelson.monkey.org 70')
        self.assertEqual('/~nelson/weblog/index.rss091', m.group('url'))
        self.assertEqual('20/Mar/2005', m.group('date'))
        self.assertEqual('07:00:49', m.group('time'))
        self.assertEqual('-0800', m.group('tz'))

if __name__ == '__main__':
    if len(sys.argv) == 1:
        unittest.main()
    else:
        for f in sys.argv[1:]:
            dumpMatches(f)