From ffbf497620e919d0c1861cb52b33de21a0b2d8c4 Mon Sep 17 00:00:00 2001 From: Joseph Myers Date: Wed, 20 Jun 2018 21:14:51 +0000 Subject: [PATCH 63/64] Python 3 preparation: update tokenize use in cgitb.py. --- roundup/cgi/cgitb.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/roundup/cgi/cgitb.py b/roundup/cgi/cgitb.py index a5ad9cf..e8bfc0c 100644 --- a/roundup/cgi/cgitb.py +++ b/roundup/cgi/cgitb.py @@ -11,6 +11,7 @@ import sys, os, keyword, linecache, tokenize, inspect, cgi import pydoc, traceback from roundup.cgi import templating, TranslationService +from roundup.anypy.strings import s2b def get_translator(i18n=None): """Return message translation function (gettext) @@ -156,12 +157,23 @@ def html(context=5, i18n=None): names.append(token) if type == tokenize.NEWLINE: raise IndexError def linereader(file=file, lnum=[lnum]): - line = linecache.getline(file, lnum[0]) + line = s2b(linecache.getline(file, lnum[0])) lnum[0] = lnum[0] + 1 return line + # The interface that is tokenize.tokenize in Python 3 is + # called tokenize.generate_tokens in Python 2. However, + # Python 2 has tokenize.tokenize with a different interface, + # and Python 3 has an undocumented generate_tokens function, + # also with a different interface, so a version check is + # needed instead of checking for which functions exist. + if sys.version_info[0] > 2: + tokenize_fn = tokenize.tokenize + else: + tokenize_fn = tokenize.generate_tokens try: - tokenize.tokenize(linereader, tokeneater) + for t in tokenize_fn(linereader): + tokeneater(*t) except IndexError: pass lvals = [] -- 2.7.4