Merge pull request #11 from floe/master

Parse new SF XML export format

1 files changed, 52 insertions, 41 deletions

diff --git a/issues.py b/issues.py
index 2fc9a24..deeb741 100755
--- a/issues.py
+++ b/issues.py
@@ -21,12 +21,14 @@ except (ValueError, IndexError):
 if opts.github_user:
     github_user = opts.github_user
 
-from BeautifulSoup import BeautifulStoneSoup
+from bs4 import BeautifulSoup
+from datetime import datetime
 
 print 'Parsing XML export...'
-soup = BeautifulStoneSoup(open(xml_file_name, 'r'), convertEntities=BeautifulStoneSoup.ALL_ENTITIES)
+soup = BeautifulSoup(open(xml_file_name, 'r'), ['lxml'])
+#convertEntities=BeautifulStoneSoup.ALL_ENTITIES)
 
-trackers = soup.document.find('trackers', recursive=False).findAll('tracker', recursive=False)
+trackers = soup.find_all('artifact')
 
 from time import sleep
 from getpass import getpass
@@ -73,57 +75,69 @@ def labelify(string):
 
 closed_status_ids = set()
 for tracker in trackers:
-    for status in tracker.statuses('status', recursive=False):
-        status_id = status.id.string
-        status_name = status.nameTag.string
-        if status_name in ['Closed', 'Deleted']:
-            closed_status_ids.add(status_id)
+    status = tracker.find('field',attrs={'name':'status'})
+    status_id = status.parent.find('field',attrs={'name':'artifact_id'}).string
+    status_name = status.string
+    if status_name in ['Closed', 'Deleted']:
+        closed_status_ids.add(status_id)
 print "closed_status_ids:", closed_status_ids
 
-groups = {}
+groups = set()
 for tracker in trackers:
-    for group in tracker.groups('group', recursive=False):
-        groups[group.id.string] = group.group_name.string
+    group = tracker.find('field',attrs={'name':'artifact_type'})
+    groups.add(group.string)
 print "groups:", groups
 
-categories = {}
-for category in tracker.categories('category', recursive=False):
-    categories[category.id.string] = category.category_name.string
+categories = set()
+for tracker in trackers:
+    category = tracker.find('field',attrs={'name':'category'})
+    categories.add(category.string)
 print "categories:", categories
 
 def handle_tracker_item(item, issue_title_prefix):
     if len(issue_title_prefix) > 0:
         issue_title_prefix = issue_title_prefix.strip() + " "
-        
-    title = issue_title_prefix + item.summary.string
+
+    title = item.find('field',attrs={'name':'summary'}).string
+    item_id = item.find('field',attrs={'name':'artifact_id'}).string
+    item_submitter = item.find('field',attrs={'name':'submitted_by'}).string
+    item_details = item.find('field',attrs={'name':'details'}).string
+    item_date = datetime.fromtimestamp(float(item.find('field',attrs={'name':'open_date'}).string))
     body = '\n\n'.join([
-        'Converted from [SourceForge issue %s](%s), submitted by %s' % (item.id.string, item.url.string, item.submitter.string),
-        item.details.string,
+        'Submitted by %s on %s' % (item_submitter, str(item_date)),
+        item_details,
     ])
-    closed = item.status_id.string in closed_status_ids
+    closed = item_id in closed_status_ids
     labels = []
     try:
-        labels.append(labelify(groups[item.group_id.string]))
+        if "Feature" in issue_title_prefix:
+          labels.append("enhancement")
+        labels.append("import")
+        labels.append(labelify("sf#"+item_id))
     except KeyError:
         pass
     try:
-        labels.append(labelify(categories[item.category_id.string]))
+        category = labelify(item.find('field',attrs={'name':'category'}).string)
+        if category != "none":
+          labels.append(category)
     except KeyError:
         pass
 
     comments = []
-    for followup in item.followups('followup', recursive=False):
-        comments.append('\n\n'.join([
-            'Submitted by %s' % followup.submitter.string,
-            followup.details.string,
+    messages = item.findAll('message',recursive=True)
+    for followup in messages:
+        commentdate = datetime.fromtimestamp(float(followup.find('field',attrs={'name':'adddate'}).string))
+        comments.insert(0,'\n\n'.join([
+            'Submitted by %s on %s' % (followup.find('field',attrs={'name':'user_name'}).string,str(commentdate)),
+            followup.find('field',attrs={'name':'body'}).string,
         ]))
 
-    print 'Creating: %s [%s] (%d comments)%s for SF #%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item.id.string)
+    print 'Creating: %s [%s] (%d comments)%s for SF #%s from %s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item_id, item_date)
     response = rest_call('POST', 'issues', {'title': title, 'body': body})
     if response.status_code == 500:
         print "ISSUE CAUSED SERVER SIDE ERROR AND WAS NOT SAVED!!! Import will continue."
     else:
-        issue = response.json()
+        issue = json.loads(response.content)
         if 'number' not in issue:
             raise RuntimeError("No 'number' in issue; response %d invalid" % response.status_code)
         number = issue['number']
@@ -186,7 +200,7 @@ def userVerify(txt, abortOnFail=True):
 
 def getIssueTitlePrefix(trackername):
     prefixes = {
-        "Bug": "",
+        "Bugs": "",
         "Feature Request": "[Feature]",
         "Patch": "[Patch]",
         "Tech Support": "[Support]"
@@ -208,24 +222,21 @@ skipped_count = 0
 started = opts.start_id is None
 items = []
 for tracker in trackers:
-    trackeritems = tracker.tracker_items('tracker_item', recursive=False)
-    trackername = tracker.description.string
-    print "Found tracker:", trackername, ", ", len(trackeritems), "items"
+    trackername = tracker.find('field',attrs={'name':'artifact_type'}).string
     trackername = trackername.replace("Tracking System", "")
     trackername = trackername.strip()
     
     issue_title_prefix = None
-    for item in trackeritems:
-        if not started:
-            if item.id.string == opts.start_id:
-                started = True
-            else:
-                skipped_count += 1
-                continue
+    if not started:
+        if tracker.find('field',attrs={'name':'artifact_id'}).string == opts.start_id:
+            started = True
+        else:
+            skipped_count += 1
+            continue
 
-        if issue_title_prefix is None:
-            issue_title_prefix = getIssueTitlePrefix(trackername)
-        items.append((item, issue_title_prefix))
+    if issue_title_prefix is None:
+        issue_title_prefix = getIssueTitlePrefix(trackername)
+    items.append((tracker, issue_title_prefix))
 
 print "Found", len(items), "items (" + str(skipped_count) + " skipped) in", len(trackers), "trackers."