diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | README | 43 | ||||
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | better_exchook.py | 211 | ||||
| -rwxr-xr-x | issues.py | 131 |
5 files changed, 335 insertions, 61 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc @@ -1,43 +0,0 @@ -sf2github README -================ - -`sf2github` is a Python program that reads an XML export from a SourceForge project and pushes this data to GitHub via its REST API. - -The script is currently very incomplete and barely tested. If it works for you, great; if not, fix it up and send me a pull request! Currently, only migration of tracker issues is partly implemented, and there's no error handling. - -Also note that the GitHub API is quite slow, taking about 5 seconds per request on my machine and internet connection. Migration of a large project will take a while. - -Issue migration ---------------- - -What works (for me): - -* SF tracker issues become GitHub tracker issues. -* Comments on SF become comments in GitHub. -* Groups and categories on SF both become labels on GitHub. -* Issues with a status that is exactly the text "Closed" or "Deleted" will be closed on GitHub. - -Limitations: - -* Only a single tracker is supported, though this could be easily fixed. -* All issues and comments will be owned by the project's owner on GitHub, but mention the SF username of the original submitter. -* There's some rubbish in the comment text sometimes (Logged In, user_id, Originator) but this is in the SF XML export. -* There are encoding errors in the SF export of (at least) comments. Non-ASCII characters are encoded with UTF-8, then decoded (interpreted) as CP1252, and those code points gets encoded as XML entities. The script does not work around this. See also http://stackoverflow.com/questions/5291081/how-did-sourceforge-maim-this-unicode-character - -Code migration --------------- - -This script doesn't help you to migrate code from SF's Subversion to GitHub. However, I found the following page helpful in doing that: http://help.github.com/svn-importing/ - -Usage ------ - -Run the `issues.py` script and it will print instructions. Basically, if your SF XML export is in `foo.xml`, your GitHub username is `john` and your repository is `bar`: - - ./issues.py foo.xml john/bar - -License -------- - -This software is in the public domain. I accept no responsibility for any damage resulting from it. Use at your own risk. - @@ -22,11 +22,19 @@ Limitations: * Only a single tracker is supported, though this could be easily fixed. * All issues and comments will be owned by the project's owner on GitHub, but mention the SF username of the original submitter. * There's some rubbish in the comment text sometimes (Logged In, user_id, Originator) but this is in the SF XML export. +* There are encoding errors in the SF export of (at least) comments. Non-ASCII characters are encoded with UTF-8, then decoded (interpreted) as CP1252, and those code points gets encoded as XML entities. The script does not work around this. See also http://stackoverflow.com/questions/5291081/how-did-sourceforge-maim-this-unicode-character + +Code migration +-------------- + +This script doesn't help you to migrate code from SF's Subversion to GitHub. However, I found the following page helpful in doing that: http://help.github.com/svn-importing/ Usage ----- -Run the `issues.py` script and it will print instructions. Basically, if your SF XML export is in `foo.xml`, your GitHub username is `john` and your repository is `bar`: +From SourceForge, you need to export the tracker data in XML. Read [here](https://sourceforge.net/apps/trac/sourceforge/wiki/XML%20export) for instructions. + +Run the `issues.py` script and it will print further instructions. Basically, if your SF XML export is in `foo.xml`, your GitHub username is `john` and your repository is `bar`: ./issues.py foo.xml john/bar diff --git a/better_exchook.py b/better_exchook.py new file mode 100644 index 0000000..083cbc2 --- /dev/null +++ b/better_exchook.py @@ -0,0 +1,211 @@ + +# by Albert Zeyer, www.az2000.de +# code under GPLv3+ +# 2011-04-15 + +# This is a simple replacement for the standard Python exception handler (sys.excepthook). +# In addition to what the standard handler does, it also prints all referenced variables +# (no matter if local, global or builtin) of the code line of each stack frame. +# See below for some examples and some example output. + +import sys + +def parse_py_statement(line): + state = 0 + curtoken = "" + spaces = " \t\n" + ops = ".,;:+-*/%&=(){}[]^<>" + i = 0 + def _escape_char(c): + if c == "n": return "\n" + elif c == "t": return "\t" + else: return c + while i < len(line): + c = line[i] + i += 1 + if state == 0: + if c in spaces: pass + elif c in ops: yield ("op", c) + elif c == "#": state = 6 + elif c == "\"": state = 1 + elif c == "'": state = 2 + else: + curtoken = c + state = 3 + elif state == 1: # string via " + if c == "\\": state = 4 + elif c == "\"": + yield ("str", curtoken) + curtoken = "" + state = 0 + else: curtoken += c + elif state == 2: # string via ' + if c == "\\": state = 5 + elif c == "'": + yield ("str", curtoken) + curtoken = "" + state = 0 + else: curtoken += c + elif state == 3: # identifier + if c in spaces + ops + "#\"'": + yield ("id", curtoken) + curtoken = "" + state = 0 + i -= 1 + else: curtoken += c + elif state == 4: # escape in " + curtoken += _escape_char(c) + state = 1 + elif state == 5: # escape in ' + curtoken += _escape_char(c) + state = 2 + elif state == 6: # comment + curtoken += c + if state == 3: yield ("id", curtoken) + elif state == 6: yield ("comment", curtoken) + +def grep_full_py_identifiers(tokens): + tokens = list(tokens) + i = 0 + pykeywords = set(["for","in","while","print","continue","break","if","else","elif","yield","def","class","try","except","import","pass","lambda"]) + while i < len(tokens): + tokentype, token = tokens[i] + i += 1 + if tokentype != "id": continue + while i+1 < len(tokens) and tokens[i] == ("op", ".") and tokens[i+1][0] == "id": + token += "." + tokens[i+1][1] + i += 2 + if token == "": continue + if token in pykeywords: continue + if token[0] in ".0123456789": continue + yield token + + +def output_limit(): + return 300 + +def output(s): + limit = output_limit() + if len(s) > limit: + s = s[:limit - 3] + "..." + sys.stderr.write(s) + sys.stderr.write("\n") + sys.stderr.flush() + +def debug_shell(user_ns, user_global_ns): + from IPython.Shell import IPShellEmbed,IPShell + ipshell = IPShell(argv=[], user_ns=user_ns, user_global_ns=user_global_ns) + #ipshell() + ipshell.mainloop() + + +def better_exchook(etype, value, tb): + output("EXCEPTION") + output('Traceback (most recent call last):') + topFrameLocals,topFrameGlobals = None,None + try: + import linecache + limit = None + if hasattr(sys, 'tracebacklimit'): + limit = sys.tracebacklimit + n = 0 + _tb = tb + def _resolveIdentifier(namespace, id): + obj = namespace[id[0]] + for part in id[1:]: + obj = getattr(obj, part) + return obj + def _trySet(old, func): + if old is not None: return old + try: return func() + except: return old + while _tb is not None and (limit is None or n < limit): + f = _tb.tb_frame + topFrameLocals,topFrameGlobals = f.f_locals,f.f_globals + lineno = _tb.tb_lineno + co = f.f_code + filename = co.co_filename + name = co.co_name + output(' File "%s", line %d, in %s' % (filename,lineno,name)) + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, f.f_globals) + if line: + line = line.strip() + output(' line: ' + line) + output(' locals:') + alreadyPrintedLocals = set() + for tokenstr in grep_full_py_identifiers(parse_py_statement(line)): + splittedtoken = tuple(tokenstr.split(".")) + for token in map(lambda i: splittedtoken[0:i], range(1, len(splittedtoken) + 1)): + if token in alreadyPrintedLocals: continue + tokenvalue = None + tokenvalue = _trySet(tokenvalue, lambda: "<local> " + repr(_resolveIdentifier(f.f_locals, token))) + tokenvalue = _trySet(tokenvalue, lambda: "<global> " + repr(_resolveIdentifier(f.f_globals, token))) + tokenvalue = _trySet(tokenvalue, lambda: "<builtin> " + repr(_resolveIdentifier(f.f_builtins, token))) + tokenvalue = tokenvalue or "<not found>" + output(' ' + ".".join(token) + " = " + tokenvalue) + alreadyPrintedLocals.add(token) + if len(alreadyPrintedLocals) == 0: output(" no locals") + _tb = _tb.tb_next + n += 1 + + except Exception, e: + output("ERROR: cannot get more detailed exception info because:") + import traceback + for l in traceback.format_exc().split("\n"): output(" " + l) + output("simple traceback:") + traceback.print_tb(tb) + + import types + def _some_str(value): + try: return str(value) + except: return '<unprintable %s object>' % type(value).__name__ + def _format_final_exc_line(etype, value): + valuestr = _some_str(value) + if value is None or not valuestr: + line = "%s" % etype + else: + line = "%s: %s" % (etype, valuestr) + return line + if (isinstance(etype, BaseException) or + isinstance(etype, types.InstanceType) or + etype is None or type(etype) is str): + output(_format_final_exc_line(etype, value)) + else: + output(_format_final_exc_line(etype.__name__, value)) + + debug = False + try: + import os + debug = int(os.environ["DEBUG"]) != 0 + except: pass + if debug: + output("---------- DEBUG SHELL -----------") + debug_shell(user_ns=topFrameLocals, user_global_ns=topFrameGlobals) + +def install(): + sys.excepthook = better_exchook + +if __name__ == "__main__": + # some examples + # this code produces this output: https://gist.github.com/922622 + + try: + x = {1:2, "a":"b"} + def f(): + y = "foo" + x, 42, sys.stdin.__class__, sys.exc_info, y, z + f() + except: + better_exchook(*sys.exc_info()) + + try: + f = lambda x: None + f(x, y) + except: + better_exchook(*sys.exc_info()) + + # use this to overwrite the global exception handler + sys.excepthook = better_exchook + # and fail + finalfail(sys) @@ -1,5 +1,8 @@ #!/usr/bin/env python +import better_exchook +better_exchook.install() + import sys import optparse @@ -20,10 +23,6 @@ print 'Parsing XML export...' soup = BeautifulStoneSoup(open(xml_file_name, 'r'), convertEntities=BeautifulStoneSoup.ALL_ENTITIES) trackers = soup.document.find('trackers', recursive=False).findAll('tracker', recursive=False) -if len(trackers) > 1: - print 'Multiple trackers not yet supported, sorry' - sys.exit(1) -tracker = trackers[0] from urllib import urlencode from urllib2 import Request, urlopen @@ -32,9 +31,8 @@ from time import sleep from getpass import getpass import re -github_password = getpass('%s\'s GitHub password: ' % github_user) - def rest_call(before, after, data_dict=None): + global github_user, github_password url = 'https://github.com/api/v2/xml/%s/%s/%s' % (before, github_repo, after) if data_dict is None: data = None @@ -52,29 +50,39 @@ def rest_call(before, after, data_dict=None): def labelify(string): return re.sub(r'[^a-z0-9._-]+', '-', string.lower()) -closed_status_ids = [] -for status in tracker.statuses('status', recursive=False): - status_id = status.id.string - status_name = status.nameTag.string - if status_name in ['Closed', 'Deleted']: - closed_status_ids.append(status_id) +closed_status_ids = set() +for tracker in trackers: + for status in tracker.statuses('status', recursive=False): + status_id = status.id.string + status_name = status.nameTag.string + if status_name in ['Closed', 'Deleted']: + closed_status_ids.add(status_id) +print "closed_status_ids:", closed_status_ids groups = {} -for group in tracker.groups('group', recursive=False): - groups[group.id.string] = group.group_name.string +for tracker in trackers: + for group in tracker.groups('group', recursive=False): + groups[group.id.string] = group.group_name.string +print "groups:", groups categories = {} for category in tracker.categories('category', recursive=False): categories[category.id.string] = category.category_name.string +print "categories:", categories started = opts.start_id is None -for item in tracker.tracker_items('tracker_item', recursive=False): +def handle_tracker_item(item, issue_title_prefix): + global started if not started: if item.id.string == opts.start_id: started = True else: - continue - title = item.summary.string + return + + if len(issue_title_prefix) > 0: + issue_title_prefix = issue_title_prefix.strip() + " " + + title = issue_title_prefix + item.summary.string body = '\n\n'.join([ 'Converted from [SourceForge issue %s](%s), submitted by %s' % (item.id.string, item.url.string, item.submitter.string), item.details.string, @@ -111,3 +119,92 @@ for item in tracker.tracker_items('tracker_item', recursive=False): print 'Closing...' rest_call('issues/close', number) + +import signal +def signal_handler(signal, frame): + print 'You pressed Ctrl+C!' + import sys + sys.exit(0) +signal.signal(signal.SIGINT, signal_handler) + +import readline +readline.parse_and_bind("tab: complete") +readline.parse_and_bind("set show-all-if-ambiguous on") + +class Completer: + def __init__(self, words): + self.words = words + self.prefix = None + + def complete(self, prefix, index): + if prefix != self.prefix: + self.matching_words = [w for w in self.words if w.startswith(prefix)] + self.prefix = prefix + else: + pass + try: + return self.matching_words[index] + except IndexError: + return None + +def userRawInput(prompt): + readline.set_completer(None) + s = raw_input(prompt) + return s + +def userInput(words, prompt=""): + readline.set_completer(Completer(words).complete) + while True: + s = raw_input((prompt + " ").lstrip() + "Choice of [" + ", ".join(words) + "] ? ") + if s in words: return s + print "Error: '" + s + "' unknown, please try again" + +def userVerify(txt, abortOnFail=True): + if userInput(["yes","no"], txt) != 'yes': + if abortOnFail: + print "Aborted." + sys.exit(1) + return False + return True + +def getIssueTitlePrefix(trackername): + prefixes = { + "Bug": "", + "Feature Request": "[Feature]", + "Patch": "[Patch]", + "Tech Support": "[Support]" + } + if trackername in prefixes: + return prefixes[trackername] + + prefix = "[" + trackername + "]" + if not userVerify("Tracker '" + trackername + "' is unknown," + + "I would use the prefix '" + prefix + "', ok?", False): + + while True: + prefix = userRawInput("Please enter a prefix: ") + if userVerify("Is prefix '" + prefix + "' ok?"): + break + return prefix + +items = [] +for tracker in trackers: + trackeritems = tracker.tracker_items('tracker_item', recursive=False) + trackername = tracker.description.string + print "Found tracker:", trackername, ",", len(trackeritems), "items" + trackername = trackername.replace("Tracking System", "") + trackername = trackername.strip() + + issue_title_prefix = None + for item in trackeritems: + if issue_title_prefix is None: + issue_title_prefix = getIssueTitlePrefix(trackername) + items.append((item, issue_title_prefix)) + +print "Found", len(items), "items in", len(trackers), "trackers." + +userVerify("Everything ok, should I really start?") +github_password = getpass('%s\'s GitHub password: ' % github_user) +for item, issue_title_prefix in items: + handle_tracker_item(item, issue_title_prefix) + |