workitems.py: request larger batch from +specs list

[bin.git] / workitems.py

diff --git a/workitems.py b/workitems.py
index 8572a75961dd5fd505f65a817e8a868b02ee14f6..d30a7e68386752fa0f2db0e611ece170048d274d 100755 (executable)
--- a/workitems.py
+++ b/workitems.py
@@ -41,6 +41,9 @@ def parse_argv():
         help='Regex pattern for blueprint name', dest='pattern')
     optparser.add_option('-D', '--dump', action='store_true', default=False,
         help='Dump database', dest='dump')
+    optparser.add_option('-m', '--moin', metavar='URL',
+        help='moin URL for additional work items (can be given multiple times)', 
+        action='append', dest='moin', default=[])
     optparser.add_option('-t', '--text', action='store_true', default=False,
         help='Print work item summary in text format', dest='text')
     optparser.add_option('-c', '--csv', action='store_true', default=False,
@@ -76,7 +79,7 @@ def get_blueprints(url, name_pattern):
 
     return result
 
-def get_workitems(blueprint_url):
+def get_blueprint_workitems(blueprint_url):
     '''Collect work items from a particular blueprint URL.
 
     This will return a list of ('item', 'status') pairs.
@@ -106,6 +109,10 @@ def get_workitems(blueprint_url):
         state = state.strip().lower()
         if not state:
             state = 'todo'
+        if state == 'completed':
+            state = 'done'
+        if state == 'inprogress':
+            state = 'todo'
         if state not in valid_states:
             print >> sys.stderr, 'ERROR: invalid state "%s" for work item "%s"' % (
                 state, desc)
@@ -114,6 +121,31 @@ def get_workitems(blueprint_url):
 
     return result
 
+def get_moin_workitems(url):
+    '''Collect work items from a moin wiki URL.
+
+    Every line starting with "|| " is treated as a work item.
+
+    Return a list of ('item', 'status') pairs.
+    '''
+    result = []
+    for line in urllib.urlopen(url):
+        if line.startswith('|| '):
+            fields = line.strip().split('||')
+            assert not fields[0] # should be empty
+            desc = fields[1].strip()
+            for f in fields[2:]:
+                if 'DONE' in f:
+                    result.append((desc, 'done'))
+                    break
+                elif 'POSTPONED' in f:
+                    result.append((desc, 'done'))
+                    break
+            else:
+                result.append((desc, 'todo'))
+
+    return result
+
 def dump(db):
     '''Dump database contents.'''
 
@@ -132,7 +164,7 @@ def add_work_item(db, blueprint, item, status):
 def import_lp(db, name_pattern, release):
     '''Collect blueprint work items from Launchpad into DB.'''
 
-    blueprints = get_blueprints('%s//ubuntu/%s/+specs' % (blueprints_base_url,
+    blueprints = get_blueprints('%s//ubuntu/%s/+specs?batch=300' % (blueprints_base_url,
         opts.release), name_pattern)
 
     cur = db.cursor()
@@ -141,12 +173,11 @@ def import_lp(db, name_pattern, release):
     for bp in blueprints:
         #print 'Checking', bp
         bpname = bp.split('/')[-1]
-        work_items = get_workitems(bp)
+        work_items = get_blueprint_workitems(bp)
         if not work_items:
             print >> sys.stderr, 'WARNING: %s has no work items' % bpname
         for (item, status) in work_items:
             add_work_item(db, bpname, item, status)
-    db.commit()
 
 def workitems_over_time(db):
     '''Calculate work item development over time.
@@ -230,6 +261,13 @@ def csv(db, from_date, to_date):
                 entry.get('postponed', 0))
         d += datetime.timedelta(days=1)
 
+def import_moin(db, urls):
+    '''Collect blueprint work items from a moin wiki.'''
+
+    for url in urls:
+        for (d, s) in get_moin_workitems(url):
+            add_work_item(db, url, d, s)
+
 #
 # main
 #
@@ -246,4 +284,6 @@ elif opts.csv:
     csv(db, opts.from_date, opts.to_date)
 else:
     import_lp(db, opts.pattern, opts.release)
+    import_moin(db, opts.moin)
+    db.commit()