3 import urllib, re, sys, optparse, os.path, datetime
4 import sqlite3 as dbapi2
6 blueprints_base_url = 'https://blueprints.launchpad.net'
8 valid_states = ['todo', 'done', 'postponed']
11 '''Open/initialize database.
13 This creates the database if it does not exist.
15 init = not os.path.exists(dbpath)
17 db = dbapi2.connect(dbpath)
21 cur.execute('''CREATE TABLE work_items (
22 blueprint VARCHAR(255) NOT NULL,
23 workitem VARCHAR(255) NOT NULL,
24 status VARCHAR(20) NOT NULL,
25 date TIMESTAMP NOT NULL)''')
26 cur.execute('''CREATE TABLE status (
27 blueprint VARCHAR(255) NOT NULL,
28 status VARCHAR(1000) NOT NULL,
29 date TIMESTAMP NOT NULL)''')
35 '''Parse CLI arguments.
37 Return (options, args) tuple.
39 optparser = optparse.OptionParser()
40 optparser.add_option('-d', '--database',
41 help='Path to database', dest='database', metavar='PATH')
42 optparser.add_option('-r', '--release',
43 help='Release name', dest='release')
44 optparser.add_option('-p', '--pattern', metavar='REGEX',
45 help='Regex pattern for blueprint name', dest='pattern')
46 optparser.add_option('-D', '--dump', action='store_true', default=False,
47 help='Dump database', dest='dump')
48 optparser.add_option('-m', '--moin', metavar='URL',
49 help='moin URL for additional work items (can be given multiple times)',
50 action='append', dest='moin', default=[])
51 optparser.add_option('-t', '--text', action='store_true', default=False,
52 help='Print work item summary in text format', dest='text')
53 optparser.add_option('-c', '--csv', action='store_true', default=False,
54 help='Print work item summary in CSV format', dest='csv')
55 optparser.add_option('-H', '--html', action='store_true', default=False,
56 help='Generate work item HTML report', dest='html')
57 optparser.add_option('--from', metavar='YYYY-MM-DD',
58 help='Generate CSV data from this day on', dest='from_date')
59 optparser.add_option('--to', metavar='YYYY-MM-DD',
60 help='Generate CSV data until this day', dest='to_date')
62 (opts, args) = optparser.parse_args()
65 optparser.error('No database given')
66 if not opts.dump and not opts.text and not opts.csv and not opts.html:
68 optparser.error('No release given')
70 optparser.error('No pattern given')
74 def get_blueprints(url, name_pattern):
75 '''Return a list of blueprint URLs for the current release.'''
77 blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
81 for l in urllib.urlopen(url):
82 m = blueprint_name_filter.search(l)
84 result.append(blueprints_base_url + m.group(1))
88 def get_blueprint_workitems(blueprint_url):
89 '''Collect work items from a particular blueprint URL.
91 This will return a list of ('item', 'status') pairs.
93 work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
95 found_workitems = False
97 for l in urllib.urlopen(blueprint_url):
99 if not found_workitems:
100 if work_items_re.search(l):
101 found_workitems = True
106 l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
112 (desc, state) = l.rsplit(':', 1)
114 print >> sys.stderr, 'ERROR: invalid work item format: ' + l
117 state = state.strip().lower()
120 if state == 'completed':
122 if state == 'inprogress':
124 if state == 'postpone':
126 if state not in valid_states:
127 print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
130 result.append((desc, state))
137 def get_blueprint_status(blueprint_url):
138 '''Collect status from a particular blueprint URL.
140 This will return a list of lines.
142 status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
146 for l in urllib.urlopen(blueprint_url):
149 if status_re.search(l):
155 l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
160 result.append(l.strip())
165 return "\n".join(result)
167 def get_moin_workitems(url):
168 '''Collect work items from a moin wiki URL.
170 Every line starting with "|| " is treated as a work item.
172 Return a list of ('item', 'status') pairs.
175 for line in urllib.urlopen(url):
176 if line.startswith('|| '):
177 fields = line.strip().split('||')
178 assert not fields[0] # should be empty
179 desc = fields[1].strip()
182 result.append((desc, 'done'))
184 elif 'POSTPONED' in f:
185 result.append((desc, 'done'))
188 result.append((desc, 'todo'))
193 '''Dump database contents.'''
196 cur.execute('SELECT * FROM work_items')
197 print '== Work items: =='
198 for (blueprint, workitem, item_status, date) in cur:
199 print '%s [%s]\t%s: %s' % (date, blueprint, workitem, item_status)
201 print '\n== Status =='
203 cur.execute('SELECT * FROM status')
204 for (blueprint, status, date) in cur:
205 print '%s: %s [%s]' % (blueprint, status, date)
207 def add_work_item(db, blueprint, item, status):
208 '''Add work item to database.'''
211 cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
212 (blueprint, item, status))
214 def add_status(db, blueprint, status):
215 '''Add blueprint status to database.'''
221 cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
224 def import_lp(db, name_pattern, release):
225 '''Collect blueprint work items and status from Launchpad into DB.'''
227 blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
228 opts.release), name_pattern)
231 cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
232 cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
234 for bp in blueprints:
235 #print 'Checking', bp
236 bpname = bp.split('/')[-1]
237 work_items = get_blueprint_workitems(bp)
238 status = get_blueprint_status(bp)
240 print >> sys.stderr, 'WARNING: %s has no work items' % bpname
241 for (item, state) in work_items:
242 add_work_item(db, bpname, item, state)
243 add_status(db, bpname, status)
245 def workitems_over_time(db):
246 '''Calculate work item development over time.
248 Return date -> state -> count mapping.
251 for s in valid_states:
253 cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
255 for (date, num) in cur:
256 data.setdefault(date, {})[s] = num
259 def blueprint_completion(db):
260 '''Determine current blueprint completion.
262 Return blueprint -> [todo, done, postponed, status] mapping.
268 cur.execute('SELECT max(date) FROM work_items')
269 (last_date,) = cur.fetchone()
272 for s in valid_states:
274 cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
275 'LEFT JOIN status s ON w.blueprint = s.blueprint '
276 'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
278 for (bp, num, status) in cur:
279 data.setdefault(bp, [0, 0, 0, ''])[index] = num
280 data[bp][-1] = status or ''
286 '''Print work item completion as text.'''
288 data = workitems_over_time(db)
291 for d in sorted(data.keys()):
294 print '\nBlueprint completion:'
295 data = blueprint_completion(db)
296 for (bp, (todo, done, postponed, status)) in data.iteritems():
298 print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed,
299 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
301 def csv(db, from_date, to_date):
302 '''Print work item status as csv.'''
305 '''Convert datetime.date into MM/DD/YYYY'''
307 return '%s/%s/%s' % (d.month, d.day, d.year)
310 '''Convert YYYY-MM-DD string to datetime.date'''
312 (y, m, d) = s.split('-')
313 return datetime.date(int(y), int(m), int(d))
315 data = workitems_over_time(db)
317 dates = sorted(data.keys())
319 f = _fromstr(from_date or dates[0])
320 t = _fromstr(to_date or dates[-1])
324 entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
325 print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year,
326 entry.get('todo', 0), entry.get('done', 0),
327 entry.get('postponed', 0))
328 d += datetime.timedelta(days=1)
331 '''Print work item status as HTML.'''
335 <title>Work item status</title>
336 <style type="text/css">
337 body { background: #CCCCB0; color: black; }
338 a { text-decoration: none; }
339 table { border-collapse: collapse; border-style: solid none;
340 border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
341 table th { text-align: left; border-style: none none solid none;
342 border-width: 3px; padding-right: 10px; }
343 table td { text-align: left; border-style: none none dotted none;
344 border-width: 1px; padding-right: 10px; }
353 <p><img src="burndown.png" alt="burndown" /></p>
355 <h1>Status by blueprint</h1>
357 <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
360 data = blueprint_completion(db)
363 for (bp, (todo, done, postponed, status)) in data.iteritems():
364 completion.append((bp,
365 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
367 completion.sort(key=lambda k: k[1], reverse=True)
369 for (bp, percent) in completion:
370 print ' <tr><td><a href="%s/ubuntu/+spec/%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
371 blueprints_base_url, bp, bp, data[bp][0], data[bp][2],
372 data[bp][1], percent,
377 print '</body></html>'
379 def import_moin(db, urls):
380 '''Collect blueprint work items from a moin wiki.'''
383 for (d, s) in get_moin_workitems(url):
384 add_work_item(db, url, d, s)
390 (opts, args) = parse_argv()
392 db = get_db(opts.database)
401 csv(db, opts.from_date, opts.to_date)
403 import_lp(db, opts.pattern, opts.release)
404 import_moin(db, opts.moin)