workitems.py: add support for status: field, thanks to Loïc Minier!

diff --combined workitems.py
index d5621062e2a58e07da80ac6575bf17b9cfcaeeaf,928c941b1e0f969f5761527f2657690fef720b5b..7b3ebef577b2319564588f455c2f4e2108bfd994
--- 1/workitems.py
--- 2/workitems.py
+++ b/workitems.py
@@@ -23,6 -23,10 +23,10 @@@ def get_db(dbpath)
              workitem VARCHAR(255) NOT NULL,
              status VARCHAR(20) NOT NULL,
              date TIMESTAMP NOT NULL)''')
 -        cur.execute('''CREATE TABLE statuses (
++        cur.execute('''CREATE TABLE status (
+             blueprint VARCHAR(255) NOT NULL,
 -            status VARCHAR(20) NOT NULL,
++            status VARCHAR(1000) NOT NULL,
+             date TIMESTAMP NOT NULL)''')
          db.commit()
  
      return db
@@@ -84,7 -88,7 +88,7 @@@ def get_blueprints(url, name_pattern)
  def get_blueprint_workitems(blueprint_url):
      '''Collect work items from a particular blueprint URL.
  
 -    This will return a list of ('item', 'state') pairs.
 +    This will return a list of ('item', 'status') pairs.
      '''
      work_items_re = re.compile('(<p>|^)work items:\s*<br />', re.I)
  
@@@ -130,12 -134,42 +134,42 @@@
  
      return result
  
 -    for l in urllib.urlopen(blueprint_url):
+ def get_blueprint_status(blueprint_url):
+     '''Collect status from a particular blueprint URL.
+ 
+     This will return a list of lines.
+     '''
+     status_re = re.compile('(<p>|^)status:\s*<br />', re.I)
+ 
+     found_status = False
+     result = []
++    for l in urllib.urlopen(blueprint_url + '?'):
+         end = False
+         if not found_status:
+             if status_re.search(l):
+                 found_status = True
+             continue
+ 
+         if '</p>' in l:
+             end = True
+         l = l.replace('<br />', '').replace('</div>', '').replace('</p>', '').strip()
+ 
+         if not l:
+             break
+ 
+         result.append(l.strip())
+ 
+         if end:
+             break
+ 
+     return "\n".join(result)
+ 
  def get_moin_workitems(url):
      '''Collect work items from a moin wiki URL.
  
      Every line starting with "|| " is treated as a work item.
  
 -    Return a list of ('item', 'state') pairs.
 +    Return a list of ('item', 'status') pairs.
      '''
      result = []
      for line in urllib.urlopen(url):
@@@ -159,34 -193,46 +193,54 @@@ def dump(db)
      '''Dump database contents.'''
  
      cur = db.cursor()
 -    cur.execute('SELECT w.*, s.status FROM work_items w LEFT JOIN statuses s on w.blueprint = s.blueprint')
 -    for (blueprint, workitem, status, date) in cur:
 -        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
 +    cur.execute('SELECT * FROM work_items')
-     for (blueprint, workitem, status, date) in cur:
-         print '%s [%s]\t%s: %s' % (date, blueprint, workitem, status)
++    print '== Work items: =='
++    for (blueprint, workitem, item_status, date) in cur:
++        print '%s [%s]\t%s: %s' % (date, blueprint, workitem, item_status)
+ 
 -def add_work_item(db, blueprint, item, state):
++    print '\n== Status =='
++    cur = db.cursor()
++    cur.execute('SELECT * FROM status')
++    for (blueprint, status, date) in cur:
++        print '%s: %s [%s]' % (blueprint, status, date)
 +
 +def add_work_item(db, blueprint, item, status):
      '''Add work item to database.'''
  
      cur = db.cursor()
      cur.execute('INSERT INTO work_items VALUES (?, ?, ?, date(CURRENT_TIMESTAMP))',
 -            (blueprint, item, state))
 +            (blueprint, item, status))
  
 -    cur.execute('INSERT INTO statuses VALUES (?, ?, date(CURRENT_TIMESTAMP))',
+ def add_status(db, blueprint, status):
+     '''Add blueprint status to database.'''
+ 
++    if not status:
++        return
++
+     cur = db.cursor()
++    cur.execute('INSERT INTO status VALUES (?, ?, date(CURRENT_TIMESTAMP))',
+             (blueprint, status))
+ 
  def import_lp(db, name_pattern, release):
-     '''Collect blueprint work items from Launchpad into DB.'''
+     '''Collect blueprint work items and status from Launchpad into DB.'''
  
      blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
          opts.release), name_pattern)
  
      cur = db.cursor()
      cur.execute('DELETE FROM work_items WHERE date = date(CURRENT_TIMESTAMP)')
 -    cur.execute('DELETE FROM statuses WHERE date = date(CURRENT_TIMESTAMP)')
++    cur.execute('DELETE FROM status WHERE date = date(CURRENT_TIMESTAMP)')
  
      for bp in blueprints:
          #print 'Checking', bp
          bpname = bp.split('/')[-1]
          work_items = get_blueprint_workitems(bp)
+         status = get_blueprint_status(bp)
          if not work_items:
              print >> sys.stderr, 'WARNING: %s has no work items' % bpname
-         for (item, status) in work_items:
-             add_work_item(db, bpname, item, status)
 -        if not status:
 -            print >> sys.stderr, 'WARNING: %s has no status' % bpname
+         for (item, state) in work_items:
+             add_work_item(db, bpname, item, state)
+         add_status(db, bpname, status)
  
  def workitems_over_time(db):
      '''Calculate work item development over time.
@@@ -202,10 -248,10 +256,10 @@@
              data.setdefault(date, {})[s] = num
      return data
  
- def blueprint_status(db):
-     '''Determine current blueprint status.
 -def blueprint_stats(db):
 -    '''Determine current blueprint stats.
++def blueprint_completion(db):
++    '''Determine current blueprint completion.
  
--    Return blueprint -> [todo, done, postponed] mapping.
++    Return blueprint -> [todo, done, postponed, status] mapping.
      '''
      data = {}
  
@@@ -217,17 -263,19 +271,19 @@@
      index = 0
      for s in valid_states:
          cur = db.cursor()
-         cur.execute('SELECT blueprint, count(workitem) FROM work_items '
-                 'WHERE status = ? AND date = ? GROUP BY blueprint', 
+         cur.execute('SELECT w.blueprint, count(w.workitem), s.status FROM work_items w '
 -                'LEFT JOIN statuses s ON w.blueprint = s.blueprint '
++                'LEFT JOIN status s ON w.blueprint = s.blueprint '
+                 'WHERE w.status = ? AND w.date = ? GROUP BY w.blueprint',
                  (s, last_date))
-         for (bp, num) in cur:
-             data.setdefault(bp, [0, 0, 0])[index] = num
+         for (bp, num, status) in cur:
 -            data.setdefault(bp, [0, 0, 0, ""])[index] = num
 -            data[bp][-1] = status
++            data.setdefault(bp, [0, 0, 0, ''])[index] = num
++            data[bp][-1] = status or ''
          index += 1
  
      return data
  
  def text(db):
-     '''Print work item status as text.'''
 -    '''Print work item stats as text.'''
++    '''Print work item completion as text.'''
  
      data = workitems_over_time(db)
  
@@@ -235,9 -283,10 +291,10 @@@
      for d in sorted(data.keys()):
          print d, data[d]
  
-     print '\nBlueprint status:'
-     data = blueprint_status(db)
-     for (bp, (todo, done, postponed)) in data.iteritems():
 -    print '\nBlueprint stats:'
 -    data = blueprint_stats(db)
++    print '\nBlueprint completion:'
++    data = blueprint_completion(db)
+     for (bp, (todo, done, postponed, status)) in data.iteritems():
+         # TODO print status
          print '%s: %i/%i (%i%%)' % (bp, postponed+done, todo+done+postponed, 
                  int(float(postponed+done)/(todo+done+postponed)*100 + 0.5))
  
@@@ -297,22 -346,23 +354,23 @@@ def html(db)
  
  <h1>Status by blueprint</h1>
  <table>
-   <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th></tr>
+   <tr><th>Blueprint</th> <th>todo/postponed/done</th> <th>Completion</th> <th>Status</th></tr>
  '''
  
-     data = blueprint_status(db)
 -    data = blueprint_stats(db)
++    data = blueprint_completion(db)
  
      completion = []
-     for (bp, (todo, done, postponed)) in data.iteritems():
+     for (bp, (todo, done, postponed, status)) in data.iteritems():
          completion.append((bp,
              int(float(postponed+done)/(todo+done+postponed)*100 + 0.5)))
  
      completion.sort(key=lambda k: k[1], reverse=True)
  
      for (bp, percent) in completion:
-         print '  <tr><td><a href="%s/ubuntu/+spec/%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td></tr>' % (
+         print '  <tr><td><a href="%s/ubuntu/+spec/%s">%s</a></td> <td>%i/%i/%i</td> <td>%i%%</td> <td>%s</td></tr>' % (
                  blueprints_base_url, bp, bp, data[bp][0], data[bp][2],
-                 data[bp][1], percent)
+                 data[bp][1], percent,
+                 data[bp][-1])
  
      print '</table>'