diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 213e9b46ce3..b4a58f0db26 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1109,6 +1109,10 @@ def test_exact_type(self): token.NAME, token.AMPER, token.NUMBER, token.RPAR) + def test_pathological_trailing_whitespace(self): + # See http://bugs.python.org/issue16152 + self.assertExactTypeEqual('@ ', token.AT) + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index d669412db3e..cbf91ef222c 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -162,7 +162,7 @@ def maybe(*choices): return group(*choices) + '?' group("'", r'\\\r?\n'), StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) def _compile(expr): @@ -555,6 +555,8 @@ def _tokenize(readline, encoding): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if (initial in numchars or # ordinary number diff --git a/Misc/ACKS b/Misc/ACKS index bc5fc613b98..7f2903345d4 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -77,6 +77,7 @@ Ulf Bartelt Don Bashford Pior Bastida Nick Bastin +Ned Batchelder Jeff Bauer Michael R Bax Anthony Baxter diff --git a/Misc/NEWS b/Misc/NEWS index 5cbf5feab1d..8f11fc66f08 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -91,6 +91,9 @@ Core and Builtins Library ------- +- Issue #16152: fix tokenize to ignore whitespace at the end of the code when + no newline is found. Patch by Ned Batchelder. + - Issue #16284: Prevent keeping unnecessary references to worker functions in concurrent.futures ThreadPoolExecutor.