import urllib, re, sys, optparse, os.path, datetime
import sqlite3 as dbapi2
+from xml.sax.saxutils import escape
blueprints_base_url = 'https://blueprints.launchpad.net'
-valid_states = set(['todo', 'done', 'postponed'])
+valid_states = ['todo', 'done', 'postponed']
def get_db(dbpath):
'''Open/initialize database.
cur = db.cursor()
cur.execute('''CREATE TABLE work_items (
blueprint VARCHAR(255) NOT NULL,
- workitem VARCHAR(255) NOT NULL,
+ workitem VARCHAR(1000) NOT NULL,
status VARCHAR(20) NOT NULL,
+ assignee VARCHAR(200) NOT NULL,
+ date TIMESTAMP NOT NULL)''')
+ cur.execute('''CREATE TABLE status (
+ blueprint VARCHAR(255) NOT NULL,
+ status VARCHAR(1000) NOT NULL,
date TIMESTAMP NOT NULL)''')
db.commit()
help='Path to database', dest='database', metavar='PATH')
optparser.add_option('-r', '--release',
help='Release name', dest='release')
+ optparser.add_option('--milestone',
+ help='Milestone name', dest='milestone')
optparser.add_option('-p', '--pattern', metavar='REGEX',
help='Regex pattern for blueprint name', dest='pattern')
optparser.add_option('-D', '--dump', action='store_true', default=False,
help='Dump database', dest='dump')
+ optparser.add_option('-m', '--moin', metavar='URL',
+ help='moin URL for additional work items (can be given multiple times)',
+ action='append', dest='moin', default=[])
optparser.add_option('-t', '--text', action='store_true', default=False,
help='Print work item summary in text format', dest='text')
optparser.add_option('-c', '--csv', action='store_true', default=False,
- help='Print work item summary in text format', dest='csv')
+ help='Print work item summary in CSV format', dest='csv')
+ optparser.add_option('-H', '--html', action='store_true', default=False,
+ help='Generate work item HTML report', dest='html')
optparser.add_option('--from', metavar='YYYY-MM-DD',
help='Generate CSV data from this day on', dest='from_date')
optparser.add_option('--to', metavar='YYYY-MM-DD',
if not opts.database:
optparser.error('No database given')
- if not opts.dump and not opts.text and not opts.csv:
+ if not opts.dump and not opts.text and not opts.csv and not opts.html:
if not opts.release:
optparser.error('No release given')
if not opts.pattern:
return (opts, args)
-def get_blueprints(url, name_pattern):
+def get_blueprints(url, name_pattern, milestone):
'''Return a list of blueprint URLs for the current release.'''
blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
name_pattern)
result = []
+ scan_tr_end = False
+ found_ms = False
+ bp = None
for l in urllib.urlopen(url):
- m = blueprint_name_filter.search(l)
- if m:
- result.append(blueprints_base_url + m.group(1))
+ if scan_tr_end:
+ if milestone:
+ if ('/+milestone/%s"' % milestone) in l:
+ found_ms = True
+ if '</tr>' in l:
+ scan_tr_end = False
+ if bp and (not milestone or found_ms):
+ result.append(bp)
+ bp = None
+ found_ms = False
+ else:
+ m = blueprint_name_filter.search(l)
+ if m:
+ bp = blueprints_base_url + m.group(1)
+ scan_tr_end = True
return result
-def get_workitems(blueprint_url):
+def get_blueprint_workitems(blueprint_url):
'''Collect work items from a particular blueprint URL.
- This will return a list of ('item', 'status') pairs.
+ This will return a list of ('item', 'status', 'assignee') tuples.
'''
work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
+ assignee_re = re.compile('<a href="https://.*launchpad.net/~([a-zA-Z0-9_-]+)" class=".*person">')
found_workitems = False
+ found_assignee = False
result = []
+ default_assignee = 'nobody'
for l in urllib.urlopen(blueprint_url):
+ end = False
+
+ if '<dt>Assignee:' in l:
+ found_assignee = True
+ continue
+
+ if found_assignee and not found_workitems:
+ m = assignee_re.search(l)
+ if m:
+ default_assignee = m.group(1)
+ found_assignee = False
+
if not found_workitems:
if work_items_re.search(l):
found_workitems = True
continue
- l = l.replace('<br />', '').replace('</div>', '').strip()
+ if '</p>' in l:
+ end = True
+ l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').replace('<wbr></wbr>', '').strip()
- # ends with empty line
- if l.endswith('</p>') or not l:
+ if not l:
break
try:
state = state.strip().lower()
if not state:
state = 'todo'
+ if state == 'completed':
+ state = 'done'
+ if state == 'inprogress':
+ state = 'todo'
+ if state == 'postpone':
+ state = 'postponed'
if state not in valid_states:
print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
state, desc)
continue
- result.append((desc, state))
+
+ if desc.startswith('['):
+ try:
+ off = desc.index(']')
+ assignee = desc[1:off]
+ desc = desc[off+1:].strip()
+ except ValueError:
+ print >> sys.stderr, 'ERROR: missing closing "]" for assignee for work item "%s"' % desc
+ else:
+ assignee = default_assignee
+
+ result.append((desc, state, assignee))
+
+ if end:
+ break
+
+ return result
+
+def get_blueprint_status(blueprint_url):
+ '''Collect status from a particular blueprint URL.
+
+ This will return a list of lines.
+ '''
+ status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
+
+ found_status = False
+ result = []
+ for l in urllib.urlopen(blueprint_url):
+ end = False
+ if not found_status:
+ if status_re.search(l):
+ found_status = True
+ continue
+
+ if '</p>' in l:
+ end = True
+ l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
+
+ if not l:
+ break
+
+ result.append(l.strip())
+
+ if end:
+ break
+
+ return "\n".join(result)
+
+def get_moin_workitems(url):
+ '''Collect work items from a moin wiki URL.
+
+ Every line starting with "|| " is treated as a work item.
+
+ Return a list of ('item', 'status') pairs.
+ '''
+ result = []
+ for line in urllib.urlopen(url):
+ if line.startswith('|| '):
+ fields = line.strip().split('||')
+ assert not fields[0] # should be empty
+ desc = fields[1].strip()
+ for f in fields[2:]:
+ if 'DONE' in f:
+ result.append((desc, 'done'))
+ break
+ elif 'POSTPONED' in f:
+ result.append((desc, 'done'))
+ break
+ else:
+ result.append((desc, 'todo'))
return result
cur = db.cursor()
cur.execute('SELECT * FROM work_items')
- for (blueprint, workitem, status, date) in cur:
- print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
+ print '== Work items: =='
+ for (blueprint, workitem, item_status, assignee, date) in cur:
+ print '%s [%s, %s]\t%s: %s' % (date, blueprint, assignee, workitem, item_status)
+
+ print '\n== Status =='
+ cur = db.cursor()
+ cur.execute('SELECT * FROM status')
+ for (blueprint, status, date) in cur:
+ print '%s: %s [%s]' % (blueprint, status, date)
-def add_work_item(db, blueprint, item, status):
+def add_work_item(db, blueprint, item, status, assignee):
'''Add work item to database.'''
cur = db.cursor()
- cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
- (blueprint, item, status))
+ cur.execute('INSERT INTO work_items VALUES (?, ?, ?, ?, date(CURRENT_TIMESTAMP))',
+ (blueprint, item, status, assignee))
+
+def add_status(db, blueprint, status):
+ '''Add blueprint status to database.'''
+
+ if not status:
+ return
+
+ cur = db.cursor()
+ cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
+ (blueprint, status))
-def import_lp(db, name_pattern, release):
- '''Collect blueprint work items from Launchpad into DB.'''
+def import_lp(db, name_pattern, release, milestone):
+ '''Collect blueprint work items and status from Launchpad into DB.'''
- blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
- opts.release), name_pattern)
+ blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
+ release), name_pattern, milestone)
+
+ cur = db.cursor()
+ cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
+ cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
for bp in blueprints:
#print 'Checking', bp
bpname = bp.split('/')[-1]
- work_items = get_workitems(bp)
+ work_items = get_blueprint_workitems(bp)
+ status = get_blueprint_status(bp)
if not work_items:
print >> sys.stderr, 'WARNING: %s has no work items' % bpname
- for (item, status) in work_items:
- add_work_item(db, bpname, item, status)
- db.commit()
+ for (item, state, assignee) in work_items:
+ add_work_item(db, bpname, item, state, assignee)
+ add_status(db, bpname, status)
def workitems_over_time(db):
'''Calculate work item development over time.
data.setdefault(date, {})[s] = num
return data
-def blueprint_status(db):
- '''Determine current blueprint status.
+def blueprint_completion(db):
+ '''Determine current blueprint completion.
- Return blueprint -> [todo, done, postponed] mapping.
+ Return blueprint -> [todo, done, postponed, status] mapping.
'''
data = {}
index = 0
for s in valid_states:
cur = db.cursor()
- cur.execute('SELECT blueprint, count(workitem) FROM work_items '
- 'WHERE status = ? AND date = ? GROUP BY blueprint',
+ cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
+ 'LEFT JOIN status s ON w.blueprint = s.blueprint '
+ 'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
(s, last_date))
- for (bp, num) in cur:
- data.setdefault(bp, [0, 0, 0])[index] = num
+ for (bp, num, status) in cur:
+ data.setdefault(bp, [0, 0, 0, ''])[index] = num
+ data[bp][-1] = status or ''
+ index += 1
+
+ return data
+
+def assignee_completion(db):
+ '''Determine current by-assignee completion.
+
+ Return assignee -> [todo, done, postponed] mapping.
+ '''
+ data = {}
+
+ # last date
+ cur = db.cursor()
+ cur.execute('SELECT max(date) FROM work_items')
+ (last_date,) = cur.fetchone()
+
+ index = 0
+ for s in valid_states:
+ cur = db.cursor()
+ cur.execute('SELECT assignee, count(workitem) FROM work_items '
+ 'WHERE date=? and status=? GROUP BY assignee',
+ (last_date, s))
+ for (a, num) in cur:
+ data.setdefault(a, [0, 0, 0])[index] = num
index += 1
return data
def text(db):
- '''Print work item status as text.'''
+ '''Print work item completion as text.'''
data = workitems_over_time(db)
for d in sorted(data.keys()):
print d, data[d]
- print '\nBlueprint status:'
- data = blueprint_status(db)
- for (bp, (todo, done, postponed)) in data.iteritems():
+ print '\nBlueprint completion:'
+ data = blueprint_completion(db)
+ for (bp, (todo, done, postponed, status)) in data.iteritems():
+ # TODO print status
print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed,
int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
data = workitems_over_time(db)
dates = sorted(data.keys())
+ if not dates:
+ return
f = _fromstr(from_date or dates[0])
t = _fromstr(to_date or dates[-1])
entry.get('postponed', 0))
d += datetime.timedelta(days=1)
+def html(db):
+ '''Print work item status as HTML.'''
+
+ print '''<html>
+<head>
+ <title>Work item status</title>
+ <style type="text/css">
+ body { background: #CCCCB0; color: black; }
+ a { text-decoration: none; }
+ table { border-collapse: collapse; border-style: solid none;
+ border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
+ table th { text-align: left; border-style: none none solid none;
+ border-width: 3px; padding-right: 10px; }
+ table td { text-align: left; border-style: none none dotted none;
+ border-width: 1px; padding-right: 10px; }
+
+ a { color: blue; }
+ </style>
+</head>
+
+<body>
+
+<h1>History</h1>
+<p><img src="burndown.png" alt="burndown" /></p>
+
+<h1>Status by blueprint</h1>
+<table>
+ <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
+'''
+
+ data = blueprint_completion(db)
+
+ completion = []
+ for (bp, (todo, done, postponed, status)) in data.iteritems():
+ completion.append((bp,
+ int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
+
+ completion.sort(key=lambda k: k[1], reverse=True)
+
+ for (bp, percent) in completion:
+ if bp.startswith('http:') or bp.startswith('https:'):
+ url = bp
+ else:
+ url = '%s/ubuntu/+spec/%s' % (blueprints_base_url, escape(bp))
+ print ' <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
+ url, escape(bp), data[bp][0], data[bp][2],
+ data[bp][1], percent,
+ escape(data[bp][-1]))
+
+ print '</table>'
+
+ print '''
+<h1>Status by assignee</h1>
+<table>
+ <tr><th>Assignee</th> <th>todo/postponed/done</th> <th>Completion</th></tr>
+'''
+ data = assignee_completion(db)
+
+ completion = []
+ for (a, (todo, done, postponed)) in data.iteritems():
+ completion.append((a,
+ int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
+
+ completion.sort(key=lambda k: k[0], reverse=False)
+
+ for (a, percent) in completion:
+ url = '%s/~%s/+specs?role=assignee' % (blueprints_base_url, a)
+ print ' <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td></tr>' % (
+ url, escape(a), data[a][0], data[a][2],
+ data[a][1], percent)
+ print '</table>'
+
+ print '</body></html>'
+
+def import_moin(db, urls):
+ '''Collect blueprint work items from a moin wiki.'''
+
+ for url in urls:
+ for (d, s) in get_moin_workitems(url):
+ add_work_item(db, url, d, s, 'nobody')
+
#
# main
#
dump(db)
elif opts.text:
text(db)
+elif opts.html:
+ html(db)
elif opts.csv:
csv(db, opts.from_date, opts.to_date)
else:
- import_lp(db, opts.pattern, opts.release)
+ import_lp(db, opts.pattern, opts.release, opts.milestone)
+ import_moin(db, opts.moin)
+ db.commit()
projects / bin.git / blobdiff