diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 506aa6a42f9..f575e9bc237 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -24,6 +24,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' 'Michael Foord') +import builtins import re import sys from token import * @@ -335,13 +336,11 @@ def find_cookie(line): return default, [first, second] -_builtin_open = open - def open(filename): """Open a file in read only mode using the encoding detected by detect_encoding(). """ - buffer = _builtin_open(filename, 'rb') + buffer = builtins.open(filename, 'rb') encoding, lines = detect_encoding(buffer.readline) buffer.seek(0) text = TextIOWrapper(buffer, encoding, line_buffering=True) diff --git a/Misc/NEWS b/Misc/NEWS index 39abedcd83d..a5ac7c104e1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,8 @@ Core and Builtins Library ------- +- Issue #11074: Make 'tokenize' so it can be reloaded. + - Issue #4681: Allow mmap() to work on file sizes and offsets larger than 4GB, even on 32-bit builds. Initial patch by Ross Lagerwall, adapted for 32-bit Windows.