X-Git-Url: https://piware.de/gitweb/?a=blobdiff_plain;f=workitems.py;h=fa40e47764b0800d579b0f252df41171a2e1d27c;hb=1a473459cc1097ae568d5dc32bce05e16d2a38bb;hp=8fa15cb777bad5df73e82ee6b3eb68aca0f9a579;hpb=79ce1657e7f890c8f28fdd3b018fd4d2bd7135d8;p=bin.git

diff --git a/workitems.py b/workitems.py
index 8fa15cb..fa40e47 100755
--- a/workitems.py
+++ b/workitems.py
@@ -5,7 +5,7 @@ import sqlite3 as dbapi2
 
 blueprints_base_url = 'https://blueprints.launchpad.net'
 
-valid_states = set(['todo', 'done', 'postponed'])
+valid_states = ['todo', 'done', 'postponed']
 
 def get_db(dbpath):
     '''Open/initialize database.
@@ -23,6 +23,10 @@ def get_db(dbpath):
             workitem VARCHAR(255) NOT NULL,
             status VARCHAR(20) NOT NULL,
             date TIMESTAMP NOT NULL)''')
+        cur.execute('''CREATE TABLE status (
+            blueprint VARCHAR(255) NOT NULL,
+            status VARCHAR(1000) NOT NULL,
+            date TIMESTAMP NOT NULL)''')
         db.commit()
 
     return db
@@ -41,10 +45,15 @@ def parse_argv():
         help='Regex pattern for blueprint name', dest='pattern')
     optparser.add_option('-D', '--dump', action='store_true', default=False,
         help='Dump database', dest='dump')
+    optparser.add_option('-m', '--moin', metavar='URL',
+        help='moin URL for additional work items (can be given multiple times)', 
+        action='append', dest='moin', default=[])
     optparser.add_option('-t', '--text', action='store_true', default=False,
         help='Print work item summary in text format', dest='text')
     optparser.add_option('-c', '--csv', action='store_true', default=False,
-        help='Print work item summary in text format', dest='csv')
+        help='Print work item summary in CSV format', dest='csv')
+    optparser.add_option('-H', '--html', action='store_true', default=False,
+        help='Generate work item HTML report', dest='html')
     optparser.add_option('--from', metavar='YYYY-MM-DD',
         help='Generate CSV data from this day on', dest='from_date')
     optparser.add_option('--to', metavar='YYYY-MM-DD',
@@ -54,7 +63,7 @@ def parse_argv():
 
     if not opts.database:
         optparser.error('No database given')
-    if not opts.dump and not opts.text and not opts.csv:
+    if not opts.dump and not opts.text and not opts.csv and not opts.html:
         if not opts.release:
             optparser.error('No release given')
         if not opts.pattern:
@@ -76,7 +85,7 @@ def get_blueprints(url, name_pattern):
 
     return result
 
-def get_workitems(blueprint_url):
+def get_blueprint_workitems(blueprint_url):
     '''Collect work items from a particular blueprint URL.
 
     This will return a list of ('item', 'status') pairs.
@@ -86,17 +95,17 @@ def get_workitems(blueprint_url):
     found_workitems = False
     result = []
     for l in urllib.urlopen(blueprint_url):
+        end = False
         if not found_workitems:
             if work_items_re.search(l):
                 found_workitems = True
             continue
 
-        l = l.strip()
-        if l.endswith('<br />'):
-            l = l[:-6]
+        if '</p>' in l:
+            end = True
+        l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
 
-        # ends with empty line
-        if l.endswith('</p>') or not l:
+        if not l:
             break
 
         try:
@@ -108,12 +117,76 @@ def get_workitems(blueprint_url):
         state = state.strip().lower()
         if not state:
             state = 'todo'
+        if state == 'completed':
+            state = 'done'
+        if state == 'inprogress':
+            state = 'todo'
+        if state == 'postpone':
+            state = 'postponed'
         if state not in valid_states:
             print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
                 state, desc)
             continue
         result.append((desc, state))
 
+        if end:
+            break
+
+    return result
+
+def get_blueprint_status(blueprint_url):
+    '''Collect status from a particular blueprint URL.
+
+    This will return a list of lines.
+    '''
+    status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
+
+    found_status = False
+    result = []
+    for l in urllib.urlopen(blueprint_url):
+        end = False
+        if not found_status:
+            if status_re.search(l):
+                found_status = True
+            continue
+
+        if '</p>' in l:
+            end = True
+        l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
+
+        if not l:
+            break
+
+        result.append(l.strip())
+
+        if end:
+            break
+
+    return "\n".join(result)
+
+def get_moin_workitems(url):
+    '''Collect work items from a moin wiki URL.
+
+    Every line starting with "|| " is treated as a work item.
+
+    Return a list of ('item', 'status') pairs.
+    '''
+    result = []
+    for line in urllib.urlopen(url):
+        if line.startswith('|| '):
+            fields = line.strip().split('||')
+            assert not fields[0] # should be empty
+            desc = fields[1].strip()
+            for f in fields[2:]:
+                if 'DONE' in f:
+                    result.append((desc, 'done'))
+                    break
+                elif 'POSTPONED' in f:
+                    result.append((desc, 'done'))
+                    break
+            else:
+                result.append((desc, 'todo'))
+
     return result
 
 def dump(db):
@@ -121,8 +194,15 @@ def dump(db):
 
     cur = db.cursor()
     cur.execute('SELECT * FROM work_items')
-    for (blueprint, workitem, status, date) in cur:
-        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
+    print '== Work items: =='
+    for (blueprint, workitem, item_status, date) in cur:
+        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, item_status)
+
+    print '\n== Status =='
+    cur = db.cursor()
+    cur.execute('SELECT * FROM status')
+    for (blueprint, status, date) in cur:
+        print '%s: %s [%s]' % (blueprint, status, date)
 
 def add_work_item(db, blueprint, item, status):
     '''Add work item to database.'''
@@ -131,21 +211,36 @@ def add_work_item(db, blueprint, item, status):
     cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
             (blueprint, item, status))
 
+def add_status(db, blueprint, status):
+    '''Add blueprint status to database.'''
+
+    if not status:
+        return
+
+    cur = db.cursor()
+    cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
+            (blueprint, status))
+
 def import_lp(db, name_pattern, release):
-    '''Collect blueprint work items from Launchpad into DB.'''
+    '''Collect blueprint work items and status from Launchpad into DB.'''
 
-    blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
+    blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
         opts.release), name_pattern)
 
+    cur = db.cursor()
+    cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
+    cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
+
     for bp in blueprints:
         #print 'Checking', bp
         bpname = bp.split('/')[-1]
-        work_items = get_workitems(bp)
+        work_items = get_blueprint_workitems(bp)
+        status = get_blueprint_status(bp)
         if not work_items:
             print >> sys.stderr, 'WARNING: %s has no work items' % bpname
-        for (item, status) in work_items:
-            add_work_item(db, bpname, item, status)
-    db.commit()
+        for (item, state) in work_items:
+            add_work_item(db, bpname, item, state)
+        add_status(db, bpname, status)
 
 def workitems_over_time(db):
     '''Calculate work item development over time.
@@ -161,10 +256,10 @@ def workitems_over_time(db):
             data.setdefault(date, {})[s] = num
     return data
 
-def blueprint_status(db):
-    '''Determine current blueprint status.
+def blueprint_completion(db):
+    '''Determine current blueprint completion.
 
-    Return blueprint -> [todo, done, postponed] mapping.
+    Return blueprint -> [todo, done, postponed, status] mapping.
     '''
     data = {}
 
@@ -176,17 +271,19 @@ def blueprint_status(db):
     index = 0
     for s in valid_states:
         cur = db.cursor()
-        cur.execute('SELECT blueprint, count(workitem) FROM work_items '
-                'WHERE status = ? AND date = ? GROUP BY blueprint', 
+        cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
+                'LEFT JOIN status s ON w.blueprint = s.blueprint '
+                'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
                 (s, last_date))
-        for (bp, num) in cur:
-            data.setdefault(bp, [0, 0, 0])[index] = num
+        for (bp, num, status) in cur:
+            data.setdefault(bp, [0, 0, 0, ''])[index] = num
+            data[bp][-1] = status or ''
         index += 1
 
     return data
 
 def text(db):
-    '''Print work item status as text.'''
+    '''Print work item completion as text.'''
 
     data = workitems_over_time(db)
 
@@ -194,9 +291,10 @@ def text(db):
     for d in sorted(data.keys()):
         print d, data[d]
 
-    print '\nBlueprint status:'
-    data = blueprint_status(db)
-    for (bp, (todo, done, postponed)) in data.iteritems():
+    print '\nBlueprint completion:'
+    data = blueprint_completion(db)
+    for (bp, (todo, done, postponed, status)) in data.iteritems():
+        # TODO print status
         print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed, 
                 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
 
@@ -229,6 +327,62 @@ def csv(db, from_date, to_date):
                 entry.get('postponed', 0))
         d += datetime.timedelta(days=1)
 
+def html(db):
+    '''Print work item status as HTML.'''
+
+    print '''<html>
+<head>
+  <title>Work item status</title>
+  <style type="text/css">
+    body { background: #CCCCB0; color: black; }
+    a { text-decoration: none; }
+    table { border-collapse: collapse; border-style: solid none; 
+            border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
+    table th { text-align: left; border-style: none none solid none; 
+               border-width: 3px; padding-right: 10px; }
+    table td { text-align: left; border-style: none none dotted none; 
+               border-width: 1px; padding-right: 10px; }
+
+    a { color: blue; }
+  </style>
+</head>
+
+<body>
+
+<h1>History</h1>
+<p><img src="burndown.png" alt="burndown" /></p>
+
+<h1>Status by blueprint</h1>
+<table>
+  <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
+'''
+
+    data = blueprint_completion(db)
+
+    completion = []
+    for (bp, (todo, done, postponed, status)) in data.iteritems():
+        completion.append((bp,
+            int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
+
+    completion.sort(key=lambda k: k[1], reverse=True)
+
+    for (bp, percent) in completion:
+        print '  <tr><td><a href="%s/ubuntu/+spec/%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
+                blueprints_base_url, bp, bp, data[bp][0], data[bp][2],
+                data[bp][1], percent,
+                data[bp][-1])
+
+    print '</table>'
+
+    print '</body></html>'
+
+def import_moin(db, urls):
+    '''Collect blueprint work items from a moin wiki.'''
+
+    for url in urls:
+        for (d, s) in get_moin_workitems(url):
+            add_work_item(db, url, d, s)
+
 #
 # main
 #
@@ -241,8 +395,12 @@ if opts.dump:
     dump(db)
 elif opts.text:
     text(db)
+elif opts.html:
+    html(db)
 elif opts.csv:
     csv(db, opts.from_date, opts.to_date)
 else:
     import_lp(db, opts.pattern, opts.release)
+    import_moin(db, opts.moin)
+    db.commit()