From a810a32a2059c29a28461be989eb4aea30f91c23 Mon Sep 17 00:00:00 2001
From: "martin@piware.de" <>
Date: Thu, 26 Nov 2009 12:14:51 +0100
Subject: [PATCH] remove workitems.py, moved to proper LP project

---
 workitems.py | 509 ---------------------------------------------------
 1 file changed, 509 deletions(-)
 delete mode 100755 workitems.py

diff --git a/workitems.py b/workitems.py
deleted file mode 100755
index 95f8a04..0000000
--- a/workitems.py
+++ /dev/null
@@ -1,509 +0,0 @@
-#!/usr/bin/python
-
-import urllib, re, sys, optparse, os.path, datetime
-import sqlite3 as dbapi2
-from xml.sax.saxutils import escape
-
-blueprints_base_url = 'https://blueprints.launchpad.net'
-
-valid_states = ['todo', 'done', 'postponed']
-
-def get_db(dbpath):
-    '''Open/initialize database.
-
-    This creates the database if it does not exist.
-    '''
-    init = not os.path.exists(dbpath)
-
-    db = dbapi2.connect(dbpath)
-
-    if init:
-        cur = db.cursor()
-        cur.execute('''CREATE TABLE work_items (
-            blueprint VARCHAR(255) NOT NULL,
-            workitem VARCHAR(1000) NOT NULL,
-            status VARCHAR(20) NOT NULL,
-            assignee VARCHAR(200) NOT NULL,
-            date TIMESTAMP NOT NULL)''')
-        cur.execute('''CREATE TABLE status (
-            blueprint VARCHAR(255) NOT NULL,
-            status VARCHAR(1000) NOT NULL,
-            date TIMESTAMP NOT NULL)''')
-        db.commit()
-
-    return db
-
-def parse_argv():
-    '''Parse CLI arguments.
-
-    Return (options, args) tuple.
-    '''
-    optparser = optparse.OptionParser()
-    optparser.add_option('-d', '--database',
-        help='Path to database', dest='database', metavar='PATH')
-    optparser.add_option('-r', '--release',
-        help='Release name', dest='release')
-    optparser.add_option('--milestone',
-        help='Milestone name', dest='milestone')
-    optparser.add_option('-p', '--pattern', metavar='REGEX',
-        help='Regex pattern for blueprint name', dest='pattern')
-    optparser.add_option('-D', '--dump', action='store_true', default=False,
-        help='Dump database', dest='dump')
-    optparser.add_option('-m', '--moin', metavar='URL',
-        help='moin URL for additional work items (can be given multiple times)', 
-        action='append', dest='moin', default=[])
-    optparser.add_option('-t', '--text', action='store_true', default=False,
-        help='Print work item summary in text format', dest='text')
-    optparser.add_option('-c', '--csv', action='store_true', default=False,
-        help='Print work item summary in CSV format', dest='csv')
-    optparser.add_option('-H', '--html', action='store_true', default=False,
-        help='Generate work item HTML report', dest='html')
-    optparser.add_option('--from', metavar='YYYY-MM-DD',
-        help='Generate CSV data from this day on', dest='from_date')
-    optparser.add_option('--to', metavar='YYYY-MM-DD',
-        help='Generate CSV data until this day', dest='to_date')
-
-    (opts, args) = optparser.parse_args()
-
-    if not opts.database:
-        optparser.error('No database given')
-    if not opts.dump and not opts.text and not opts.csv and not opts.html:
-        if not opts.release:
-            optparser.error('No release given')
-        if not opts.pattern:
-            optparser.error('No pattern given')
-
-    return (opts, args)
-
-def get_blueprints(url, name_pattern, milestone):
-    '''Return a list of blueprint URLs for the current release.'''
-
-    blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
-            name_pattern)
-
-    result = []
-    scan_tr_end = False
-    found_ms = False
-    bp = None
-    for l in urllib.urlopen(url):
-        if scan_tr_end:
-            if milestone:
-                if ('/+milestone/%s"' % milestone) in l:
-                    found_ms = True
-            if '</tr>' in l:
-                scan_tr_end = False
-                if bp and (not milestone or found_ms):
-                    result.append(bp)
-                bp = None
-                found_ms = False
-        else:
-            m = blueprint_name_filter.search(l)
-            if m:
-                bp = blueprints_base_url + m.group(1)
-                scan_tr_end = True
-
-    return result
-
-def get_blueprint_workitems(blueprint_url):
-    '''Collect work items from a particular blueprint URL.
-
-    This will return a list of ('item', 'status', 'assignee') tuples.
-    '''
-    work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
-    assignee_re = re.compile('<a href="https://.*launchpad.net/~([a-zA-Z0-9_-]+)" class=".*person">')
-
-    found_workitems = False
-    found_assignee = False
-    result = []
-    default_assignee = 'nobody'
-    for l in urllib.urlopen(blueprint_url):
-        end = False
-
-        if '<dt>Assignee:' in l:
-            found_assignee = True
-            continue
-
-        if found_assignee and not found_workitems:
-            m = assignee_re.search(l)
-            if m:
-                default_assignee = m.group(1)
-                found_assignee = False
-
-        if not found_workitems:
-            if work_items_re.search(l):
-                found_workitems = True
-            continue
-
-        if '</p>' in l:
-            end = True
-        l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').replace('<wbr></wbr>', '').strip()
-
-        if not l:
-            break
-
-        try:
-            (desc, state) = l.rsplit(':', 1)
-        except ValueError:
-            print >> sys.stderr, 'ERROR: invalid work item format: ' + l
-            continue
-        desc = desc.strip()
-        state = state.strip().lower()
-        if not state:
-            state = 'todo'
-        if state == 'completed':
-            state = 'done'
-        if state == 'inprogress':
-            state = 'todo'
-        if state == 'postpone':
-            state = 'postponed'
-        if state not in valid_states:
-            print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
-                state, desc)
-            continue
-
-        if desc.startswith('['):
-            try:
-                off = desc.index(']')
-                assignee = desc[1:off]
-                desc = desc[off+1:].strip()
-            except ValueError:
-                print >> sys.stderr, 'ERROR: missing closing "]" for assignee for work item "%s"' % desc
-        else:
-            assignee = default_assignee
-
-        result.append((desc, state, assignee))
-
-        if end:
-            break
-
-    return result
-
-def get_blueprint_status(blueprint_url):
-    '''Collect status from a particular blueprint URL.
-
-    This will return a list of lines.
-    '''
-    status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
-
-    found_status = False
-    result = []
-    for l in urllib.urlopen(blueprint_url):
-        end = False
-        if not found_status:
-            if status_re.search(l):
-                found_status = True
-            continue
-
-        if '</p>' in l:
-            end = True
-        l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
-
-        if not l:
-            break
-
-        result.append(l.strip())
-
-        if end:
-            break
-
-    return "\n".join(result)
-
-def get_moin_workitems(url):
-    '''Collect work items from a moin wiki URL.
-
-    Every line starting with "|| " is treated as a work item.
-
-    Return a list of ('item', 'status', 'assignee') tuples.
-    '''
-    result = []
-    for line in urllib.urlopen(url):
-        assignee = 'nobody'
-        if line.startswith('|| '):
-            fields = line.strip().split('||')
-            assert not fields[0] # should be empty
-            desc = fields[1].strip()
-            for f in fields[2:]:
-                if 'DONE' in f or 'POSTPONED' in f or 'TODO' in f or 'INPROGRESS' in f:
-                    ff = f.split()
-                    if len(ff) == 2:
-                        assignee = ff[1]
-                    if 'DONE' in f:
-                        result.append((desc, 'done', assignee))
-                        break
-                    elif 'POSTPONED' in f:
-                        result.append((desc, 'postponed', assignee))
-                        break
-                    else:
-                        result.append((desc, 'todo', assignee))
-                        break
-            else:
-                result.append((desc, 'todo', 'nobody'))
-
-    return result
-
-def dump(db):
-    '''Dump database contents.'''
-
-    cur = db.cursor()
-    cur.execute('SELECT * FROM work_items')
-    print '== Work items: =='
-    for (blueprint, workitem, item_status, assignee, date) in cur:
-        print '%s [%s, %s]\t%s: %s' % (date, blueprint, assignee, workitem, item_status)
-
-    print '\n== Status =='
-    cur = db.cursor()
-    cur.execute('SELECT * FROM status')
-    for (blueprint, status, date) in cur:
-        print '%s: %s [%s]' % (blueprint, status, date)
-
-def add_work_item(db, blueprint, item, status, assignee):
-    '''Add work item to database.'''
-
-    cur = db.cursor()
-    cur.execute('INSERT INTO work_items VALUES (?, ?, ?, ?, date(CURRENT_TIMESTAMP))',
-            (blueprint, item, status, assignee))
-
-def add_status(db, blueprint, status):
-    '''Add blueprint status to database.'''
-
-    if not status:
-        return
-
-    cur = db.cursor()
-    cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
-            (blueprint, status))
-
-def import_lp(db, name_pattern, release, milestone):
-    '''Collect blueprint work items and status from Launchpad into DB.'''
-
-    blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
-        release), name_pattern, milestone)
-
-    cur = db.cursor()
-    cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
-    cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
-
-    for bp in blueprints:
-        #print 'Checking', bp
-        bpname = bp.split('/')[-1]
-        work_items = get_blueprint_workitems(bp)
-        status = get_blueprint_status(bp)
-        if not work_items:
-            print >> sys.stderr, 'WARNING: %s has no work items' % bpname
-        for (item, state, assignee) in work_items:
-            add_work_item(db, bpname, item, state, assignee)
-        add_status(db, bpname, status)
-
-def workitems_over_time(db):
-    '''Calculate work item development over time.
-
-    Return date -> state -> count mapping.
-    '''
-    data = {}
-    for s in valid_states:
-        cur = db.cursor()
-        cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
-                (s,))
-        for (date, num) in cur:
-            data.setdefault(date, {})[s] = num
-    return data
-
-def blueprint_completion(db):
-    '''Determine current blueprint completion.
-
-    Return blueprint -> [todo, done, postponed, status] mapping.
-    '''
-    data = {}
-
-    # last date
-    cur = db.cursor()
-    cur.execute('SELECT max(date) FROM work_items')
-    (last_date,) = cur.fetchone()
-
-    index = 0
-    for s in valid_states:
-        cur = db.cursor()
-        cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
-                'LEFT JOIN status s ON w.blueprint = s.blueprint '
-                'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
-                (s, last_date))
-        for (bp, num, status) in cur:
-            data.setdefault(bp, [0, 0, 0, ''])[index] = num
-            data[bp][-1] = status or ''
-        index += 1
-
-    return data
-
-def assignee_completion(db):
-    '''Determine current by-assignee completion.
-
-    Return assignee -> [todo, done, postponed] mapping.
-    '''
-    data = {}
-
-    # last date
-    cur = db.cursor()
-    cur.execute('SELECT max(date) FROM work_items')
-    (last_date,) = cur.fetchone()
-
-    index = 0
-    for s in valid_states:
-        cur = db.cursor()
-        cur.execute('SELECT assignee, count(workitem) FROM work_items '
-                'WHERE date=? and status=? GROUP BY assignee',
-                (last_date, s))
-        for (a, num) in cur:
-            data.setdefault(a, [0, 0, 0])[index] = num
-        index += 1
-
-    return data
-
-def text(db):
-    '''Print work item completion as text.'''
-
-    data = workitems_over_time(db)
-
-    print 'History:'
-    for d in sorted(data.keys()):
-        print d, data[d]
-
-    print '\nBlueprint completion:'
-    data = blueprint_completion(db)
-    for (bp, (todo, done, postponed, status)) in data.iteritems():
-        # TODO print status
-        print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed, 
-                int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
-
-def csv(db, from_date, to_date):
-    '''Print work item status as csv.'''
-
-    def _fmtdate(d):
-        '''Convert datetime.date into MM/DD/YYYY'''
-
-        return '%s/%s/%s' % (d.month, d.day, d.year)
-
-    def _fromstr(s):
-        '''Convert YYYY-MM-DD string to datetime.date'''
-
-        (y, m, d) = s.split('-')
-        return datetime.date(int(y), int(m), int(d))
-
-    data = workitems_over_time(db)
-
-    dates = sorted(data.keys())
-    if not dates:
-        return
-
-    f = _fromstr(from_date or dates[0])
-    t = _fromstr(to_date or dates[-1])
-
-    d = f
-    while d <= t:
-        entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
-        print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year, 
-                entry.get('todo', 0), entry.get('done', 0),
-                entry.get('postponed', 0))
-        d += datetime.timedelta(days=1)
-
-def html(db):
-    '''Print work item status as HTML.'''
-
-    print '''<html>
-<head>
-  <title>Work item status</title>
-  <style type="text/css">
-    body { background: #CCCCB0; color: black; }
-    a { text-decoration: none; }
-    table { border-collapse: collapse; border-style: solid none; 
-            border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
-    table th { text-align: left; border-style: none none solid none; 
-               border-width: 3px; padding-right: 10px; }
-    table td { text-align: left; border-style: none none dotted none; 
-               border-width: 1px; padding-right: 10px; }
-
-    a { color: blue; }
-  </style>
-</head>
-
-<body>
-
-<h1>History</h1>
-<p><img src="burndown.png" alt="burndown" /></p>
-
-<h1>Status by blueprint</h1>
-<table>
-  <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
-'''
-
-    data = blueprint_completion(db)
-
-    completion = []
-    for (bp, (todo, done, postponed, status)) in data.iteritems():
-        completion.append((bp,
-            int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
-
-    completion.sort(key=lambda k: k[1], reverse=True)
-
-    for (bp, percent) in completion:
-        if bp.startswith('http:') or bp.startswith('https:'):
-            url = bp
-        else:
-            url = '%s/ubuntu/+spec/%s' % (blueprints_base_url, escape(bp))
-        print '  <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
-                url, escape(bp), data[bp][0], data[bp][2],
-                data[bp][1], percent,
-                escape(data[bp][-1]))
-
-    print '</table>'
-
-    print '''
-<h1>Status by assignee</h1>
-<table>
-  <tr><th>Assignee</th> <th>todo/postponed/done</th> <th>Completion</th></tr>
-'''
-    data = assignee_completion(db)
-
-    completion = []
-    for (a, (todo, done, postponed)) in data.iteritems():
-        completion.append((a,
-            int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
-
-    completion.sort(key=lambda k: k[0], reverse=False)
-
-    for (a, percent) in completion:
-        url = '%s/~%s/+specs?role=assignee' % (blueprints_base_url, a)
-        print '  <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td></tr>' % (
-                url, escape(a), data[a][0], data[a][2],
-                data[a][1], percent)
-    print '</table>'
-
-    print '</body></html>'
-
-def import_moin(db, urls):
-    '''Collect blueprint work items from a moin wiki.'''
-
-    for url in urls:
-        for (d, s, a) in get_moin_workitems(url):
-            add_work_item(db, url, d, s, a)
-
-#
-# main
-#
-
-(opts, args) = parse_argv()
-
-db = get_db(opts.database)
-
-if opts.dump:
-    dump(db)
-elif opts.text:
-    text(db)
-elif opts.html:
-    html(db)
-elif opts.csv:
-    csv(db, opts.from_date, opts.to_date)
-else:
-    import_lp(db, opts.pattern, opts.release, opts.milestone)
-    import_moin(db, opts.moin)
-    db.commit()
-
-- 
2.39.5