From dcdff90216206e72cc024dfa2d687d75b39ab68e Mon Sep 17 00:00:00 2001 From: Date: Sat, 27 Apr 2013 13:07:43 +0200 Subject: hackish attempt to parse the new SF XML export format --- issues.py | 79 +++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 42 insertions(+), 37 deletions(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index 2fc9a24..61582c4 100755 --- a/issues.py +++ b/issues.py @@ -26,7 +26,7 @@ from BeautifulSoup import BeautifulStoneSoup print 'Parsing XML export...' soup = BeautifulStoneSoup(open(xml_file_name, 'r'), convertEntities=BeautifulStoneSoup.ALL_ENTITIES) -trackers = soup.document.find('trackers', recursive=False).findAll('tracker', recursive=False) +trackers = soup.project_export.find('artifacts', recursive=False).findAll('artifact', recursive=False) from time import sleep from getpass import getpass @@ -73,52 +73,60 @@ def labelify(string): closed_status_ids = set() for tracker in trackers: - for status in tracker.statuses('status', recursive=False): - status_id = status.id.string - status_name = status.nameTag.string - if status_name in ['Closed', 'Deleted']: - closed_status_ids.add(status_id) + status = tracker.find('field',attrs={'name':'status'}) + status_id = status.parent.find('field',attrs={'name':'artifact_id'}).string + status_name = status.string + if status_name in ['Closed', 'Deleted']: + closed_status_ids.add(status_id) print "closed_status_ids:", closed_status_ids -groups = {} +groups = set() for tracker in trackers: - for group in tracker.groups('group', recursive=False): - groups[group.id.string] = group.group_name.string + group = tracker.find('field',attrs={'name':'artifact_type'}) + groups.add(group.string) print "groups:", groups -categories = {} -for category in tracker.categories('category', recursive=False): - categories[category.id.string] = category.category_name.string +categories = set() +for tracker in trackers: + category = tracker.find('field',attrs={'name':'category'}) + categories.add(category.string) print "categories:", categories def handle_tracker_item(item, issue_title_prefix): if len(issue_title_prefix) > 0: issue_title_prefix = issue_title_prefix.strip() + " " - - title = issue_title_prefix + item.summary.string + + title = issue_title_prefix + item.find('field',attrs={'name':'summary'}).string + item_id = item.find('field',attrs={'name':'artifact_id'}).string + item_submitter = item.find('field',attrs={'name':'submitted_by'}).string + item_details = item.find('field',attrs={'name':'details'}).string body = '\n\n'.join([ - 'Converted from [SourceForge issue %s](%s), submitted by %s' % (item.id.string, item.url.string, item.submitter.string), - item.details.string, + 'Converted from [SourceForge issue %s], submitted by %s' % (item_id, item_submitter), + item_details, ]) - closed = item.status_id.string in closed_status_ids + closed = item_id in closed_status_ids labels = [] try: - labels.append(labelify(groups[item.group_id.string])) + labels.append(labelify(item.find('field',attrs={'name':'artifact_id'}).string)) except KeyError: pass try: - labels.append(labelify(categories[item.category_id.string])) + labels.append(labelify(item.find('field',attrs={'name':'category'}).string)) except KeyError: pass comments = [] - for followup in item.followups('followup', recursive=False): + messages = item.findAll('message',recursive=True) + for followup in messages: + # workaround BeautifulSoup parsing error (?) + if len(followup.findAll('field')) == 0: + continue comments.append('\n\n'.join([ - 'Submitted by %s' % followup.submitter.string, - followup.details.string, + 'Submitted by %s' % followup.find('field',attrs={'name':'user_name'}).string, + followup.find('field',attrs={'name':'body'}).string, ])) - print 'Creating: %s [%s] (%d comments)%s for SF #%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item.id.string) + print 'Creating: %s [%s] (%d comments)%s for SF #%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item_id) response = rest_call('POST', 'issues', {'title': title, 'body': body}) if response.status_code == 500: print "ISSUE CAUSED SERVER SIDE ERROR AND WAS NOT SAVED!!! Import will continue." @@ -186,7 +194,7 @@ def userVerify(txt, abortOnFail=True): def getIssueTitlePrefix(trackername): prefixes = { - "Bug": "", + "Bugs": "", "Feature Request": "[Feature]", "Patch": "[Patch]", "Tech Support": "[Support]" @@ -208,24 +216,21 @@ skipped_count = 0 started = opts.start_id is None items = [] for tracker in trackers: - trackeritems = tracker.tracker_items('tracker_item', recursive=False) - trackername = tracker.description.string - print "Found tracker:", trackername, ", ", len(trackeritems), "items" + trackername = tracker.find('field',attrs={'name':'artifact_type'}).string trackername = trackername.replace("Tracking System", "") trackername = trackername.strip() issue_title_prefix = None - for item in trackeritems: - if not started: - if item.id.string == opts.start_id: - started = True - else: - skipped_count += 1 - continue + if not started: + if tracker.find('field',attrs={'name':'artifact_id'}).string == opts.start_id: + started = True + else: + skipped_count += 1 + continue - if issue_title_prefix is None: - issue_title_prefix = getIssueTitlePrefix(trackername) - items.append((item, issue_title_prefix)) + if issue_title_prefix is None: + issue_title_prefix = getIssueTitlePrefix(trackername) + items.append((tracker, issue_title_prefix)) print "Found", len(items), "items (" + str(skipped_count) + " skipped) in", len(trackers), "trackers." -- cgit v1.2.3 From 6c3dda4d8ab2fb46fc5fe152c848fb986d021bc5 Mon Sep 17 00:00:00 2001 From: Date: Sun, 28 Apr 2013 18:56:13 +0200 Subject: fix json parser --- issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index 61582c4..bc93864 100755 --- a/issues.py +++ b/issues.py @@ -131,7 +131,7 @@ def handle_tracker_item(item, issue_title_prefix): if response.status_code == 500: print "ISSUE CAUSED SERVER SIDE ERROR AND WAS NOT SAVED!!! Import will continue." else: - issue = response.json() + issue = json.loads(response.content) if 'number' not in issue: raise RuntimeError("No 'number' in issue; response %d invalid" % response.status_code) number = issue['number'] -- cgit v1.2.3 From c75936e5dd254b72188de7473098a51f522428f8 Mon Sep 17 00:00:00 2001 From: Date: Mon, 29 Apr 2013 01:07:08 +0200 Subject: switch xml parser to one that actually works --- issues.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index bc93864..ed9882b 100755 --- a/issues.py +++ b/issues.py @@ -21,12 +21,13 @@ except (ValueError, IndexError): if opts.github_user: github_user = opts.github_user -from BeautifulSoup import BeautifulStoneSoup +from bs4 import BeautifulSoup print 'Parsing XML export...' -soup = BeautifulStoneSoup(open(xml_file_name, 'r'), convertEntities=BeautifulStoneSoup.ALL_ENTITIES) +soup = BeautifulSoup(open(xml_file_name, 'r'), ['lxml']) +#convertEntities=BeautifulStoneSoup.ALL_ENTITIES) -trackers = soup.project_export.find('artifacts', recursive=False).findAll('artifact', recursive=False) +trackers = soup.find_all('artifact') from time import sleep from getpass import getpass @@ -121,7 +122,7 @@ def handle_tracker_item(item, issue_title_prefix): # workaround BeautifulSoup parsing error (?) if len(followup.findAll('field')) == 0: continue - comments.append('\n\n'.join([ + comments.insert(0,'\n\n'.join([ 'Submitted by %s' % followup.find('field',attrs={'name':'user_name'}).string, followup.find('field',attrs={'name':'body'}).string, ])) -- cgit v1.2.3 From fa0b9ca921259ada5737b1e05cdbc1d8c6c2d913 Mon Sep 17 00:00:00 2001 From: Date: Mon, 29 Apr 2013 17:09:22 +0200 Subject: fix "none" labels --- issues.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index ed9882b..d3e7357 100755 --- a/issues.py +++ b/issues.py @@ -112,7 +112,9 @@ def handle_tracker_item(item, issue_title_prefix): except KeyError: pass try: - labels.append(labelify(item.find('field',attrs={'name':'category'}).string)) + category = labelify(item.find('field',attrs={'name':'category'}).string) + if category != "none": + labels.append(category) except KeyError: pass -- cgit v1.2.3 From e819526aa5d14923e9b80fd6d0a834e11bf5e93d Mon Sep 17 00:00:00 2001 From: Date: Mon, 29 Apr 2013 22:19:46 +0200 Subject: put timestamps into issues/comments --- issues.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index d3e7357..0dcc3fd 100755 --- a/issues.py +++ b/issues.py @@ -22,6 +22,7 @@ if opts.github_user: github_user = opts.github_user from bs4 import BeautifulSoup +from datetime import datetime print 'Parsing XML export...' soup = BeautifulSoup(open(xml_file_name, 'r'), ['lxml']) @@ -101,14 +102,15 @@ def handle_tracker_item(item, issue_title_prefix): item_id = item.find('field',attrs={'name':'artifact_id'}).string item_submitter = item.find('field',attrs={'name':'submitted_by'}).string item_details = item.find('field',attrs={'name':'details'}).string + item_date = datetime.fromtimestamp(float(item.find('field',attrs={'name':'open_date'}).string)) body = '\n\n'.join([ - 'Converted from [SourceForge issue %s], submitted by %s' % (item_id, item_submitter), + 'Submitted by %s on %s' % (item_submitter, str(item_date)), item_details, ]) closed = item_id in closed_status_ids labels = [] try: - labels.append(labelify(item.find('field',attrs={'name':'artifact_id'}).string)) + labels.append(labelify("sf#"+item_id)) except KeyError: pass try: @@ -121,15 +123,13 @@ def handle_tracker_item(item, issue_title_prefix): comments = [] messages = item.findAll('message',recursive=True) for followup in messages: - # workaround BeautifulSoup parsing error (?) - if len(followup.findAll('field')) == 0: - continue + commentdate = datetime.fromtimestamp(float(followup.find('field',attrs={'name':'adddate'}).string)) comments.insert(0,'\n\n'.join([ - 'Submitted by %s' % followup.find('field',attrs={'name':'user_name'}).string, + 'Submitted by %s on %s' % (followup.find('field',attrs={'name':'user_name'}).string,str(commentdate)), followup.find('field',attrs={'name':'body'}).string, ])) - print 'Creating: %s [%s] (%d comments)%s for SF #%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item_id) + print 'Creating: %s [%s] (%d comments)%s for SF #%s from %s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item_id, item_date) response = rest_call('POST', 'issues', {'title': title, 'body': body}) if response.status_code == 500: print "ISSUE CAUSED SERVER SIDE ERROR AND WAS NOT SAVED!!! Import will continue." -- cgit v1.2.3 From 7fe734f2296edbb7bb0ce060634af72c5b48b7ab Mon Sep 17 00:00:00 2001 From: Date: Mon, 29 Apr 2013 22:27:41 +0200 Subject: tag feature reqs as enhancement --- issues.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'issues.py') diff --git a/issues.py b/issues.py index 0dcc3fd..deeb741 100755 --- a/issues.py +++ b/issues.py @@ -98,7 +98,7 @@ def handle_tracker_item(item, issue_title_prefix): if len(issue_title_prefix) > 0: issue_title_prefix = issue_title_prefix.strip() + " " - title = issue_title_prefix + item.find('field',attrs={'name':'summary'}).string + title = item.find('field',attrs={'name':'summary'}).string item_id = item.find('field',attrs={'name':'artifact_id'}).string item_submitter = item.find('field',attrs={'name':'submitted_by'}).string item_details = item.find('field',attrs={'name':'details'}).string @@ -110,6 +110,9 @@ def handle_tracker_item(item, issue_title_prefix): closed = item_id in closed_status_ids labels = [] try: + if "Feature" in issue_title_prefix: + labels.append("enhancement") + labels.append("import") labels.append(labelify("sf#"+item_id)) except KeyError: pass -- cgit v1.2.3