3 import urllib, re, sys, optparse, os.path, datetime
4 import sqlite3 as dbapi2
5 from xml.sax.saxutils import escape
7 blueprints_base_url = 'https://blueprints.launchpad.net'
9 valid_states = ['todo', 'done', 'postponed']
12 '''Open/initialize database.
14 This creates the database if it does not exist.
16 init = not os.path.exists(dbpath)
18 db = dbapi2.connect(dbpath)
22 cur.execute('''CREATE TABLE work_items (
23 blueprint VARCHAR(255) NOT NULL,
24 workitem VARCHAR(1000) NOT NULL,
25 status VARCHAR(20) NOT NULL,
26 assignee VARCHAR(200) NOT NULL,
27 date TIMESTAMP NOT NULL)''')
28 cur.execute('''CREATE TABLE status (
29 blueprint VARCHAR(255) NOT NULL,
30 status VARCHAR(1000) NOT NULL,
31 date TIMESTAMP NOT NULL)''')
37 '''Parse CLI arguments.
39 Return (options, args) tuple.
41 optparser = optparse.OptionParser()
42 optparser.add_option('-d', '--database',
43 help='Path to database', dest='database', metavar='PATH')
44 optparser.add_option('-r', '--release',
45 help='Release name', dest='release')
46 optparser.add_option('--milestone',
47 help='Milestone name', dest='milestone')
48 optparser.add_option('-p', '--pattern', metavar='REGEX',
49 help='Regex pattern for blueprint name', dest='pattern')
50 optparser.add_option('-D', '--dump', action='store_true', default=False,
51 help='Dump database', dest='dump')
52 optparser.add_option('-m', '--moin', metavar='URL',
53 help='moin URL for additional work items (can be given multiple times)',
54 action='append', dest='moin', default=[])
55 optparser.add_option('-t', '--text', action='store_true', default=False,
56 help='Print work item summary in text format', dest='text')
57 optparser.add_option('-c', '--csv', action='store_true', default=False,
58 help='Print work item summary in CSV format', dest='csv')
59 optparser.add_option('-H', '--html', action='store_true', default=False,
60 help='Generate work item HTML report', dest='html')
61 optparser.add_option('--from', metavar='YYYY-MM-DD',
62 help='Generate CSV data from this day on', dest='from_date')
63 optparser.add_option('--to', metavar='YYYY-MM-DD',
64 help='Generate CSV data until this day', dest='to_date')
66 (opts, args) = optparser.parse_args()
69 optparser.error('No database given')
70 if not opts.dump and not opts.text and not opts.csv and not opts.html:
72 optparser.error('No release given')
74 optparser.error('No pattern given')
78 def get_blueprints(url, name_pattern, milestone):
79 '''Return a list of blueprint URLs for the current release.'''
81 blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
88 for l in urllib.urlopen(url):
91 if ('/+milestone/%s"' % milestone) in l:
95 if bp and (not milestone or found_ms):
100 m = blueprint_name_filter.search(l)
102 bp = blueprints_base_url + m.group(1)
107 def get_blueprint_workitems(blueprint_url):
108 '''Collect work items from a particular blueprint URL.
110 This will return a list of ('item', 'status', 'assignee') tuples.
112 work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
113 assignee_re = re.compile('<a href="https://.*launchpad.net/~([a-zA-Z0-9_-]+)" class=".*person">')
115 found_workitems = False
116 found_assignee = False
118 default_assignee = 'nobody'
119 for l in urllib.urlopen(blueprint_url):
122 if '<dt>Assignee:' in l:
123 found_assignee = True
126 if found_assignee and not found_workitems:
127 m = assignee_re.search(l)
129 default_assignee = m.group(1)
130 found_assignee = False
132 if not found_workitems:
133 if work_items_re.search(l):
134 found_workitems = True
139 l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').replace('<wbr></wbr>', '').strip()
145 (desc, state) = l.rsplit(':', 1)
147 print >> sys.stderr, 'ERROR: invalid work item format: ' + l
150 state = state.strip().lower()
153 if state == 'completed':
155 if state == 'inprogress':
157 if state == 'postpone':
159 if state not in valid_states:
160 print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
164 if desc.startswith('['):
166 off = desc.index(']')
167 assignee = desc[1:off]
168 desc = desc[off+1:].strip()
170 print >> sys.stderr, 'ERROR: missing closing "]" for assignee for work item "%s"' % desc
172 assignee = default_assignee
174 result.append((desc, state, assignee))
181 def get_blueprint_status(blueprint_url):
182 '''Collect status from a particular blueprint URL.
184 This will return a list of lines.
186 status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
190 for l in urllib.urlopen(blueprint_url):
193 if status_re.search(l):
199 l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
204 result.append(l.strip())
209 return "\n".join(result)
211 def get_moin_workitems(url):
212 '''Collect work items from a moin wiki URL.
214 Every line starting with "|| " is treated as a work item.
216 Return a list of ('item', 'status', 'assignee') tuples.
219 for line in urllib.urlopen(url):
221 if line.startswith('|| '):
222 fields = line.strip().split('||')
223 assert not fields[0] # should be empty
224 desc = fields[1].strip()
226 if 'DONE' in f or 'POSTPONED' in f or 'TODO' in f or 'INPROGRESS' in f:
231 result.append((desc, 'done', assignee))
233 elif 'POSTPONED' in f:
234 result.append((desc, 'postponed', assignee))
237 result.append((desc, 'todo', assignee))
240 result.append((desc, 'todo', 'nobody'))
245 '''Dump database contents.'''
248 cur.execute('SELECT * FROM work_items')
249 print '== Work items: =='
250 for (blueprint, workitem, item_status, assignee, date) in cur:
251 print '%s [%s, %s]\t%s: %s' % (date, blueprint, assignee, workitem, item_status)
253 print '\n== Status =='
255 cur.execute('SELECT * FROM status')
256 for (blueprint, status, date) in cur:
257 print '%s: %s [%s]' % (blueprint, status, date)
259 def add_work_item(db, blueprint, item, status, assignee):
260 '''Add work item to database.'''
263 cur.execute('INSERT INTO work_items VALUES (?, ?, ?, ?, date(CURRENT_TIMESTAMP))',
264 (blueprint, item, status, assignee))
266 def add_status(db, blueprint, status):
267 '''Add blueprint status to database.'''
273 cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
276 def import_lp(db, name_pattern, release, milestone):
277 '''Collect blueprint work items and status from Launchpad into DB.'''
279 blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
280 release), name_pattern, milestone)
283 cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
284 cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
286 for bp in blueprints:
287 #print 'Checking', bp
288 bpname = bp.split('/')[-1]
289 work_items = get_blueprint_workitems(bp)
290 status = get_blueprint_status(bp)
292 print >> sys.stderr, 'WARNING: %s has no work items' % bpname
293 for (item, state, assignee) in work_items:
294 add_work_item(db, bpname, item, state, assignee)
295 add_status(db, bpname, status)
297 def workitems_over_time(db):
298 '''Calculate work item development over time.
300 Return date -> state -> count mapping.
303 for s in valid_states:
305 cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
307 for (date, num) in cur:
308 data.setdefault(date, {})[s] = num
311 def blueprint_completion(db):
312 '''Determine current blueprint completion.
314 Return blueprint -> [todo, done, postponed, status] mapping.
320 cur.execute('SELECT max(date) FROM work_items')
321 (last_date,) = cur.fetchone()
324 for s in valid_states:
326 cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
327 'LEFT JOIN status s ON w.blueprint = s.blueprint '
328 'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
330 for (bp, num, status) in cur:
331 data.setdefault(bp, [0, 0, 0, ''])[index] = num
332 data[bp][-1] = status or ''
337 def assignee_completion(db):
338 '''Determine current by-assignee completion.
340 Return assignee -> [todo, done, postponed] mapping.
346 cur.execute('SELECT max(date) FROM work_items')
347 (last_date,) = cur.fetchone()
350 for s in valid_states:
352 cur.execute('SELECT assignee, count(workitem) FROM work_items '
353 'WHERE date=? and status=? GROUP BY assignee',
356 data.setdefault(a, [0, 0, 0])[index] = num
362 '''Print work item completion as text.'''
364 data = workitems_over_time(db)
367 for d in sorted(data.keys()):
370 print '\nBlueprint completion:'
371 data = blueprint_completion(db)
372 for (bp, (todo, done, postponed, status)) in data.iteritems():
374 print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed,
375 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
377 def csv(db, from_date, to_date):
378 '''Print work item status as csv.'''
381 '''Convert datetime.date into MM/DD/YYYY'''
383 return '%s/%s/%s' % (d.month, d.day, d.year)
386 '''Convert YYYY-MM-DD string to datetime.date'''
388 (y, m, d) = s.split('-')
389 return datetime.date(int(y), int(m), int(d))
391 data = workitems_over_time(db)
393 dates = sorted(data.keys())
397 f = _fromstr(from_date or dates[0])
398 t = _fromstr(to_date or dates[-1])
402 entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
403 print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year,
404 entry.get('todo', 0), entry.get('done', 0),
405 entry.get('postponed', 0))
406 d += datetime.timedelta(days=1)
409 '''Print work item status as HTML.'''
413 <title>Work item status</title>
414 <style type="text/css">
415 body { background: #CCCCB0; color: black; }
416 a { text-decoration: none; }
417 table { border-collapse: collapse; border-style: solid none;
418 border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
419 table th { text-align: left; border-style: none none solid none;
420 border-width: 3px; padding-right: 10px; }
421 table td { text-align: left; border-style: none none dotted none;
422 border-width: 1px; padding-right: 10px; }
431 <p><img src="burndown.png" alt="burndown" /></p>
433 <h1>Status by blueprint</h1>
435 <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
438 data = blueprint_completion(db)
441 for (bp, (todo, done, postponed, status)) in data.iteritems():
442 completion.append((bp,
443 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
445 completion.sort(key=lambda k: k[1], reverse=True)
447 for (bp, percent) in completion:
448 if bp.startswith('http:') or bp.startswith('https:'):
451 url = '%s/ubuntu/+spec/%s' % (blueprints_base_url, escape(bp))
452 print ' <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
453 url, escape(bp), data[bp][0], data[bp][2],
454 data[bp][1], percent,
455 escape(data[bp][-1]))
460 <h1>Status by assignee</h1>
462 <tr><th>Assignee</th> <th>todo/postponed/done</th> <th>Completion</th></tr>
464 data = assignee_completion(db)
467 for (a, (todo, done, postponed)) in data.iteritems():
468 completion.append((a,
469 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
471 completion.sort(key=lambda k: k[0], reverse=False)
473 for (a, percent) in completion:
474 url = '%s/~%s/+specs?role=assignee' % (blueprints_base_url, a)
475 print ' <tr><td><a href="%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td></tr>' % (
476 url, escape(a), data[a][0], data[a][2],
480 print '</body></html>'
482 def import_moin(db, urls):
483 '''Collect blueprint work items from a moin wiki.'''
486 for (d, s, a) in get_moin_workitems(url):
487 add_work_item(db, url, d, s, a)
493 (opts, args) = parse_argv()
495 db = get_db(opts.database)
504 csv(db, opts.from_date, opts.to_date)
506 import_lp(db, opts.pattern, opts.release, opts.milestone)
507 import_moin(db, opts.moin)