From: martin@piware.de <>
Date: Fri, 12 Jun 2009 14:10:46 +0000 (+0200)
Subject: add workitems.py
X-Git-Url: https://piware.de/gitweb/?p=bin.git;a=commitdiff_plain;h=fe7796b53082506f99a552fe820c78ae1e8c08b0

add workitems.py
---

diff --git a/workitems.py b/workitems.py
new file mode 100755
index 0000000..301b949
--- /dev/null
+++ b/workitems.py
@@ -0,0 +1,248 @@
+#!/usr/bin/python
+
+import urllib, re, sys, optparse, os.path, datetime
+import sqlite3 as dbapi2
+
+blueprints_base_url = 'https://blueprints.launchpad.net'
+
+valid_states = set(['todo', 'done', 'postponed'])
+
+def get_db(dbpath):
+    '''Open/initialize database.
+
+    This creates the database if it does not exist.
+    '''
+    init = not os.path.exists(dbpath)
+
+    db = dbapi2.connect(dbpath)
+
+    if init:
+        cur = db.cursor()
+        cur.execute('''CREATE TABLE work_items (
+            blueprint VARCHAR(255) NOT NULL,
+            workitem VARCHAR(255) NOT NULL,
+            status VARCHAR(20) NOT NULL,
+            date TIMESTAMP NOT NULL)''')
+        db.commit()
+
+    return db
+
+def parse_argv():
+    '''Parse CLI arguments.
+
+    Return (options, args) tuple.
+    '''
+    optparser = optparse.OptionParser()
+    optparser.add_option('-d', '--database',
+        help='Path to database', dest='database', metavar='PATH')
+    optparser.add_option('-r', '--release',
+        help='Release name', dest='release')
+    optparser.add_option('-p', '--pattern', metavar='REGEX',
+        help='Regex pattern for blueprint name', dest='pattern')
+    optparser.add_option('-D', '--dump', action='store_true', default=False,
+        help='Dump database', dest='dump')
+    optparser.add_option('-t', '--text', action='store_true', default=False,
+        help='Print work item summary in text format', dest='text')
+    optparser.add_option('-c', '--csv', action='store_true', default=False,
+        help='Print work item summary in text format', dest='csv')
+    optparser.add_option('--from', metavar='YYYY-MM-DD',
+        help='Generate CSV data from this day on', dest='from_date')
+    optparser.add_option('--to', metavar='YYYY-MM-DD',
+        help='Generate CSV data until this day', dest='to_date')
+
+    (opts, args) = optparser.parse_args()
+
+    if not opts.database:
+        optparser.error('No database given')
+    if not opts.dump and not opts.text and not opts.csv:
+        if not opts.release:
+            optparser.error('No release given')
+        if not opts.pattern:
+            optparser.error('No pattern given')
+
+    return (opts, args)
+
+def get_blueprints(url, nname_pattern):
+    '''Return a list of blueprint URLs for the current release.'''
+
+    blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
+            name_pattern)
+
+    result = []
+    for l in urllib.urlopen(url):
+        m = blueprint_name_filter.search(l)
+        if m:
+            result.append(blueprints_base_url + m.group(1))
+
+    return result
+
+def get_workitems(blueprint_url):
+    '''Collect work items from a particular blueprint URL.
+
+    This will return a list of ('item', 'status') pairs.
+    '''
+    work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
+
+    found_workitems = False
+    result = []
+    for l in urllib.urlopen(blueprint_url):
+        if not found_workitems:
+            if work_items_re.search(l):
+                found_workitems = True
+            continue
+
+        l = l.strip()
+        if l.endswith('<br />'):
+            l = l[:-6]
+
+        # ends with empty line
+        if l.endswith('</p>') or not l:
+            break
+
+        try:
+            (desc, state) = l.rsplit(':', 1)
+        except ValueError:
+            print >> sys.stderr, 'ERROR: invalid work item format: ' + l
+            continue
+        desc = desc.strip()
+        state = state.strip().lower()
+        if not state:
+            state = 'todo'
+        if state not in valid_states:
+            print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
+                state, desc)
+            continue
+        result.append((desc, state))
+
+    return result
+
+def dump(db):
+    '''Dump database contents.'''
+
+    cur = db.cursor()
+    cur.execute('SELECT * FROM work_items')
+    for (blueprint, workitem, status, date) in cur:
+        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
+
+def add_work_item(db, blueprint, item, status):
+    '''Add work item to database.'''
+
+    cur = db.cursor()
+    cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
+            (blueprint, item, status))
+
+def import_lp(db, name_pattern, release):
+    '''Collect blueprint work items from Launchpad into DB.'''
+
+    blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
+        opts.release), name_pattern)
+
+    for bp in blueprints:
+        #print 'Checking', bp
+        bpname = bp.split('/')[-1]
+        work_items = get_workitems(bp)
+        if not work_items:
+            print >> sys.stderr, 'WARNING: %s has no work items' % bpname
+        for (item, status) in work_items:
+            add_work_item(db, bpname, item, status)
+    db.commit()
+
+def workitems_over_time(db):
+    '''Calculate work item development over time.
+
+    Return date -> state -> count mapping.
+    '''
+    data = {}
+    for s in valid_states:
+        cur = db.cursor()
+        cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
+                (s,))
+        for (date, num) in cur:
+            data.setdefault(date, {})[s] = num
+    return data
+
+def blueprint_status(db):
+    '''Determine current blueprint status.
+
+    Return blueprint -> [todo, done, postponed] mapping.
+    '''
+    data = {}
+
+    # last date
+    cur = db.cursor()
+    cur.execute('SELECT max(date) FROM work_items')
+    (last_date,) = cur.fetchone()
+
+    index = 0
+    for s in valid_states:
+        cur = db.cursor()
+        cur.execute('SELECT blueprint, count(workitem) FROM work_items '
+                'WHERE status = ? AND date = ? GROUP BY blueprint', 
+                (s, last_date))
+        for (bp, num) in cur:
+            data.setdefault(bp, [0, 0, 0])[index] = num
+        index += 1
+
+    return data
+
+def text(db):
+    '''Print work item status as text.'''
+
+    data = workitems_over_time(db)
+
+    print 'History:'
+    for d in sorted(data.keys()):
+        print d, data[d]
+
+    print '\nBlueprint status:'
+    data = blueprint_status(db)
+    for (bp, (todo, done, postponed)) in data.iteritems():
+        print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed, 
+                int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
+
+def csv(db, from_date, to_date):
+    '''Print work item status as csv.'''
+
+    def _fmtdate(d):
+        '''Convert datetime.date into MM/DD/YYYY'''
+
+        return '%s/%s/%s' % (d.month, d.day, d.year)
+
+    def _fromstr(s):
+        '''Convert YYYY-MM-DD string to datetime.date'''
+
+        (y, m, d) = s.split('-')
+        return datetime.date(int(y), int(m), int(d))
+
+    data = workitems_over_time(db)
+
+    dates = sorted(data.keys())
+
+    f = _fromstr(from_date or dates[0])
+    t = _fromstr(to_date or dates[-1])
+
+    d = f
+    while d <= t:
+        entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
+        print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year, 
+                entry.get('todo', 0), entry.get('done', 0),
+                entry.get('postponed', 0))
+        d += datetime.timedelta(days=1)
+
+#
+# main
+#
+
+(opts, args) = parse_argv()
+
+db = get_db(opts.database)
+
+if opts.dump:
+    dump(db)
+elif opts.text:
+    text(db)
+elif opts.csv:
+    csv(db, opts.from_date, opts.to_date)
+else:
+    import_lp(db, opts.pattern, opts.release)
+