From 8415ef60e228b81d7343c96b016ad21805bba375 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Fri, 13 May 2011 20:31:46 +0200 Subject: some more error handling. esp 413, Request Entity Too Large --- issues.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/issues.py b/issues.py index fdc2300..219ec91 100755 --- a/issues.py +++ b/issues.py @@ -25,14 +25,14 @@ soup = BeautifulStoneSoup(open(xml_file_name, 'r'), convertEntities=BeautifulSto trackers = soup.document.find('trackers', recursive=False).findAll('tracker', recursive=False) from urllib import urlencode -from urllib2 import Request, urlopen +from urllib2 import Request, urlopen, HTTPError from base64 import b64encode from time import sleep from getpass import getpass import re -def rest_call(before, after, data_dict=None): - global github_user, github_password +def __rest_call_unchecked(before, after, data_dict=None): + global github_repo, github_user, github_password url = 'https://github.com/api/v2/xml/%s/%s/%s' % (before, github_repo, after) if data_dict is None: data = None @@ -47,6 +47,21 @@ def rest_call(before, after, data_dict=None): sleep(1) return response +def rest_call(before, after, data_dict=None): + while True: + try: + return __rest_call_unchecked(before, after, data_dict) + except HTTPError, e: + print "Got HTTPError:", e + if e.code == 413 and data_dict: # Request Entity Too Large + l = max(map(len, data_dict.itervalues())) + assert l > 0 + print "Longest value has len", l, "; now we are trying with half of that" + l /= 2 + data_dict = dict(map(lambda (k,v): (k,v[0:l]), data_dict.iteritems())) + continue + raise e # reraise, we cannot handle it + def labelify(string): return re.sub(r'[^a-z0-9._-]+', '-', string.lower()) @@ -105,7 +120,7 @@ def handle_tracker_item(item, issue_title_prefix): followup.details.string, ])) - print 'Creating: %s [%s] (%d comments)%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '') + print 'Creating: %s [%s] (%d comments)%s for SF #%s' % (title, ','.join(labels), len(comments), ' (closed)' if closed else '', item.id.string) response = rest_call('issues/open', '', {'title': title, 'body': body}) issue = BeautifulStoneSoup(response, convertEntities=BeautifulStoneSoup.ALL_ENTITIES) number = issue.number.string -- cgit v1.2.3 From e59ac40fb5a862f29caa15ede18a1bcdaa24d8d5 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Fri, 13 May 2011 20:40:56 +0200 Subject: some prettier handling for start_id --- issues.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/issues.py b/issues.py index 219ec91..27f65ca 100755 --- a/issues.py +++ b/issues.py @@ -85,15 +85,7 @@ for category in tracker.categories('category', recursive=False): categories[category.id.string] = category.category_name.string print "categories:", categories -started = opts.start_id is None def handle_tracker_item(item, issue_title_prefix): - global started - if not started: - if item.id.string == opts.start_id: - started = True - else: - return - if len(issue_title_prefix) > 0: issue_title_prefix = issue_title_prefix.strip() + " " @@ -202,6 +194,8 @@ def getIssueTitlePrefix(trackername): break return prefix +skipped_count = 0 +started = opts.start_id is None items = [] for tracker in trackers: trackeritems = tracker.tracker_items('tracker_item', recursive=False) @@ -212,11 +206,18 @@ for tracker in trackers: issue_title_prefix = None for item in trackeritems: + if not started: + if item.id.string == opts.start_id: + started = True + else: + skipped_count += 1 + continue + if issue_title_prefix is None: issue_title_prefix = getIssueTitlePrefix(trackername) items.append((item, issue_title_prefix)) -print "Found", len(items), "items in", len(trackers), "trackers." +print "Found", len(items), "items (" + str(skipped_count) + " skipped) in", len(trackers), "trackers." userVerify("Everything ok, should I really start?") github_password = getpass('%s\'s GitHub password: ' % github_user) -- cgit v1.2.3 From 04ce530c9416cac5d8662d73ab231f6dee355601 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Fri, 13 May 2011 21:18:42 +0200 Subject: handle HTTP Error 500: Internal Server Error. just retry 5 times. then fail --- issues.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/issues.py b/issues.py index 27f65ca..3e56a86 100755 --- a/issues.py +++ b/issues.py @@ -48,6 +48,7 @@ def __rest_call_unchecked(before, after, data_dict=None): return response def rest_call(before, after, data_dict=None): + count500err = 0 while True: try: return __rest_call_unchecked(before, after, data_dict) @@ -60,6 +61,13 @@ def rest_call(before, after, data_dict=None): l /= 2 data_dict = dict(map(lambda (k,v): (k,v[0:l]), data_dict.iteritems())) continue + elif e.code == 500: + N = 5 + if count500err >= N: raise e + print "Waiting 10 seconds, will try", (N - count500err), "more times" + sleep(10) + count500err += 1 + continue raise e # reraise, we cannot handle it def labelify(string): -- cgit v1.2.3 From 7aad77dc6550d12f9935c7e34cc32689c1dd9ecd Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Fri, 13 May 2011 21:22:11 +0200 Subject: strip chr(13) out of comment when printing on stdout --- issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issues.py b/issues.py index 3e56a86..32a1729 100755 --- a/issues.py +++ b/issues.py @@ -128,7 +128,7 @@ def handle_tracker_item(item, issue_title_prefix): print 'Attaching label: %s' % label rest_call('issues/label/add', '%s/%s' % (label, number)) for comment in comments: - print 'Creating comment: %s' % comment[:50].replace('\n', ' ') + print 'Creating comment: %s' % comment[:50].replace('\n', ' ').replace(chr(13), '') rest_call('issues/comment', number, {'comment': comment}) if closed: print 'Closing...' -- cgit v1.2.3 From 193ec74260610929ece1a52fd2ba3924683fd773 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Fri, 13 May 2011 21:48:27 +0200 Subject: for one comment with len=509652, I was always getting a 500 error. stripping this comment by half works, though. this fixes it, i.e. no matter what the error is, if the value len is huge, we will first try to make it smaller --- issues.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/issues.py b/issues.py index 32a1729..364e2e5 100755 --- a/issues.py +++ b/issues.py @@ -54,8 +54,8 @@ def rest_call(before, after, data_dict=None): return __rest_call_unchecked(before, after, data_dict) except HTTPError, e: print "Got HTTPError:", e - if e.code == 413 and data_dict: # Request Entity Too Large - l = max(map(len, data_dict.itervalues())) + l = data_dict and max(map(len, data_dict.itervalues())) or 0 + if e.code == 413 or l >= 100000: # Request Entity Too Large assert l > 0 print "Longest value has len", l, "; now we are trying with half of that" l /= 2 -- cgit v1.2.3