3 import urllib, re, sys, optparse, os.path, datetime
4 import sqlite3 as dbapi2
6 blueprints_base_url = 'https://blueprints.launchpad.net'
8 valid_states = set(['todo', 'done', 'postponed'])
11 '''Open/initialize database.
13 This creates the database if it does not exist.
15 init = not os.path.exists(dbpath)
17 db = dbapi2.connect(dbpath)
21 cur.execute('''CREATE TABLE work_items (
22 blueprint VARCHAR(255) NOT NULL,
23 workitem VARCHAR(255) NOT NULL,
24 status VARCHAR(20) NOT NULL,
25 date TIMESTAMP NOT NULL)''')
31 '''Parse CLI arguments.
33 Return (options, args) tuple.
35 optparser = optparse.OptionParser()
36 optparser.add_option('-d', '--database',
37 help='Path to database', dest='database', metavar='PATH')
38 optparser.add_option('-r', '--release',
39 help='Release name', dest='release')
40 optparser.add_option('-p', '--pattern', metavar='REGEX',
41 help='Regex pattern for blueprint name', dest='pattern')
42 optparser.add_option('-D', '--dump', action='store_true', default=False,
43 help='Dump database', dest='dump')
44 optparser.add_option('-m', '--moin', metavar='URL',
45 help='moin URL for additional work items (can be given multiple times)',
46 action='append', dest='moin', default=[])
47 optparser.add_option('-t', '--text', action='store_true', default=False,
48 help='Print work item summary in text format', dest='text')
49 optparser.add_option('-c', '--csv', action='store_true', default=False,
50 help='Print work item summary in text format', dest='csv')
51 optparser.add_option('--from', metavar='YYYY-MM-DD',
52 help='Generate CSV data from this day on', dest='from_date')
53 optparser.add_option('--to', metavar='YYYY-MM-DD',
54 help='Generate CSV data until this day', dest='to_date')
56 (opts, args) = optparser.parse_args()
59 optparser.error('No database given')
60 if not opts.dump and not opts.text and not opts.csv:
62 optparser.error('No release given')
64 optparser.error('No pattern given')
68 def get_blueprints(url, name_pattern):
69 '''Return a list of blueprint URLs for the current release.'''
71 blueprint_name_filter = re.compile('href="(/ubuntu/\+spec/%s[^"]+)"' %
75 for l in urllib.urlopen(url):
76 m = blueprint_name_filter.search(l)
78 result.append(blueprints_base_url + m.group(1))
82 def get_blueprint_workitems(blueprint_url):
83 '''Collect work items from a particular blueprint URL.
85 This will return a list of ('item', 'status') pairs.
87 work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
89 found_workitems = False
91 for l in urllib.urlopen(blueprint_url):
92 if not found_workitems:
93 if work_items_re.search(l):
94 found_workitems = True
97 l = l.replace('<br />', '').replace('</div>', '').strip()
99 # ends with empty line
100 if l.endswith('</p>') or not l:
104 (desc, state) = l.rsplit(':', 1)
106 print >> sys.stderr, 'ERROR: invalid work item format: ' + l
109 state = state.strip().lower()
112 if state not in valid_states:
113 print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
116 result.append((desc, state))
120 def get_moin_workitems(url):
121 '''Collect work items from a moin wiki URL.
123 Every line starting with "|| " is treated as a work item.
125 Return a list of ('item', 'status') pairs.
128 for line in urllib.urlopen(url):
129 if line.startswith('|| '):
130 fields = line.strip().split('||')
131 assert not fields[0] # should be empty
132 desc = fields[1].strip()
135 result.append((desc, 'done'))
137 elif 'POSTPONED' in f:
138 result.append((desc, 'done'))
141 result.append((desc, 'todo'))
146 '''Dump database contents.'''
149 cur.execute('SELECT * FROM work_items')
150 for (blueprint, workitem, status, date) in cur:
151 print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
153 def add_work_item(db, blueprint, item, status):
154 '''Add work item to database.'''
157 cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
158 (blueprint, item, status))
160 def import_lp(db, name_pattern, release):
161 '''Collect blueprint work items from Launchpad into DB.'''
163 blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
164 opts.release), name_pattern)
167 cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
169 for bp in blueprints:
170 #print 'Checking', bp
171 bpname = bp.split('/')[-1]
172 work_items = get_blueprint_workitems(bp)
174 print >> sys.stderr, 'WARNING: %s has no work items' % bpname
175 for (item, status) in work_items:
176 add_work_item(db, bpname, item, status)
178 def workitems_over_time(db):
179 '''Calculate work item development over time.
181 Return date -> state -> count mapping.
184 for s in valid_states:
186 cur.execute('SELECT date, count(*) FROM work_items WHERE status=? GROUP BY date',
188 for (date, num) in cur:
189 data.setdefault(date, {})[s] = num
192 def blueprint_status(db):
193 '''Determine current blueprint status.
195 Return blueprint -> [todo, done, postponed] mapping.
201 cur.execute('SELECT max(date) FROM work_items')
202 (last_date,) = cur.fetchone()
205 for s in valid_states:
207 cur.execute('SELECT blueprint, count(workitem) FROM work_items '
208 'WHERE status = ? AND date = ? GROUP BY blueprint',
210 for (bp, num) in cur:
211 data.setdefault(bp, [0, 0, 0])[index] = num
217 '''Print work item status as text.'''
219 data = workitems_over_time(db)
222 for d in sorted(data.keys()):
225 print '\nBlueprint status:'
226 data = blueprint_status(db)
227 for (bp, (todo, done, postponed)) in data.iteritems():
228 print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed,
229 int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
231 def csv(db, from_date, to_date):
232 '''Print work item status as csv.'''
235 '''Convert datetime.date into MM/DD/YYYY'''
237 return '%s/%s/%s' % (d.month, d.day, d.year)
240 '''Convert YYYY-MM-DD string to datetime.date'''
242 (y, m, d) = s.split('-')
243 return datetime.date(int(y), int(m), int(d))
245 data = workitems_over_time(db)
247 dates = sorted(data.keys())
249 f = _fromstr(from_date or dates[0])
250 t = _fromstr(to_date or dates[-1])
254 entry = data.get('%i-%02i-%02i' % (d.year, d.month, d.day), {})
255 print '%02i/%02i/%i,%i,%i,%i' % (d.month, d.day, d.year,
256 entry.get('todo', 0), entry.get('done', 0),
257 entry.get('postponed', 0))
258 d += datetime.timedelta(days=1)
260 def import_moin(db, urls):
261 '''Collect blueprint work items from a moin wiki.'''
264 for (d, s) in get_moin_workitems(url):
265 add_work_item(db, url, d, s)
271 (opts, args) = parse_argv()
273 db = get_db(opts.database)
280 csv(db, opts.from_date, opts.to_date)
282 import_lp(db, opts.pattern, opts.release)
283 import_moin(db, opts.moin)