Skip to content

Commit 713379f

Browse files
committed
gh-105069: Add a readline-like callable to the tokenizer to consume input iteratively
Signed-off-by: Pablo Galindo <[email protected]>
1 parent 39f6a04 commit 713379f

File tree

7 files changed

+242
-98
lines changed

7 files changed

+242
-98
lines changed

Lib/test/test_tokenize.py

+91-50
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from test import support
22
from test.support import os_helper
3-
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
3+
from tokenize import (tokenize, untokenize, NUMBER, NAME, OP,
44
STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
55
open as tokenize_open, Untokenizer, generate_tokens,
66
NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT, TokenInfo)
@@ -50,7 +50,20 @@ def check_tokenize(self, s, expected):
5050
self.assertEqual(result,
5151
[" ENCODING 'utf-8' (0, 0) (0, 0)"] +
5252
expected.rstrip().splitlines())
53-
53+
54+
def test_invalid_readline(self):
55+
def gen():
56+
yield "sdfosdg"
57+
yield "sdfosdg"
58+
with self.assertRaises(TypeError):
59+
list(tokenize(gen().__next__))
60+
61+
def gen():
62+
yield b"sdfosdg"
63+
yield b"sdfosdg"
64+
with self.assertRaises(TypeError):
65+
list(generate_tokens(gen().__next__))
66+
5467
def test_implicit_newline(self):
5568
# Make sure that the tokenizer puts in an implicit NEWLINE
5669
# when the input lacks a trailing new line.
@@ -1154,7 +1167,8 @@ class TestTokenizerAdheresToPep0263(TestCase):
11541167

11551168
def _testFile(self, filename):
11561169
path = os.path.join(os.path.dirname(__file__), filename)
1157-
TestRoundtrip.check_roundtrip(self, open(path, 'rb'))
1170+
with open(path, 'rb') as f:
1171+
TestRoundtrip.check_roundtrip(self, f)
11581172

11591173
def test_utf8_coding_cookie_and_no_utf8_bom(self):
11601174
f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
@@ -1199,7 +1213,8 @@ def readline():
11991213
yield b''
12001214

12011215
# skip the initial encoding token and the end tokens
1202-
tokens = list(_tokenize(readline(), encoding='utf-8'))[:-2]
1216+
tokens = list(_generate_tokens_from_c_tokenizer(readline().__next__, encoding='utf-8',
1217+
extra_tokens=True))[:-2]
12031218
expected_tokens = [TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"\n')]
12041219
self.assertEqual(tokens, expected_tokens,
12051220
"bytes not decoded with encoding")
@@ -1468,13 +1483,13 @@ def test_tokenize(self):
14681483
def mock_detect_encoding(readline):
14691484
return encoding, [b'first', b'second']
14701485

1471-
def mock__tokenize(readline, encoding):
1486+
def mock__tokenize(readline, encoding, **kwargs):
14721487
nonlocal encoding_used
14731488
encoding_used = encoding
14741489
out = []
14751490
while True:
14761491
try:
1477-
next_line = next(readline)
1492+
next_line = readline()
14781493
except StopIteration:
14791494
return out
14801495
if next_line:
@@ -1491,16 +1506,16 @@ def mock_readline():
14911506
return str(counter).encode()
14921507

14931508
orig_detect_encoding = tokenize_module.detect_encoding
1494-
orig__tokenize = tokenize_module._tokenize
1509+
orig_c_token = tokenize_module._generate_tokens_from_c_tokenizer
14951510
tokenize_module.detect_encoding = mock_detect_encoding
1496-
tokenize_module._tokenize = mock__tokenize
1511+
tokenize_module._generate_tokens_from_c_tokenizer = mock__tokenize
14971512
try:
14981513
results = tokenize(mock_readline)
14991514
self.assertEqual(list(results)[1:],
15001515
[b'first', b'second', b'1', b'2', b'3', b'4'])
15011516
finally:
15021517
tokenize_module.detect_encoding = orig_detect_encoding
1503-
tokenize_module._tokenize = orig__tokenize
1518+
tokenize_module._generate_tokens_from_c_tokenizer = orig_c_token
15041519

15051520
self.assertEqual(encoding_used, encoding)
15061521

@@ -1827,12 +1842,33 @@ class CTokenizeTest(TestCase):
18271842
def check_tokenize(self, s, expected):
18281843
# Format the tokens in s in a table format.
18291844
# The ENDMARKER and final NEWLINE are omitted.
1845+
f = StringIO(s)
18301846
with self.subTest(source=s):
18311847
result = stringify_tokens_from_source(
1832-
_generate_tokens_from_c_tokenizer(s), s
1848+
_generate_tokens_from_c_tokenizer(f.readline), s
18331849
)
18341850
self.assertEqual(result, expected.rstrip().splitlines())
18351851

1852+
def test_encoding(self):
1853+
def readline(encoding):
1854+
yield "1+1".encode(encoding)
1855+
1856+
expected = [
1857+
TokenInfo(type=NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1\n'),
1858+
TokenInfo(type=OP, string='+', start=(1, 1), end=(1, 2), line='1+1\n'),
1859+
TokenInfo(type=NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1\n'),
1860+
TokenInfo(type=NEWLINE, string='\n', start=(1, 3), end=(1, 4), line='1+1\n'),
1861+
TokenInfo(type=ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
1862+
]
1863+
for encoding in ["utf-8", "latin-1", "utf-16"]:
1864+
with self.subTest(encoding=encoding):
1865+
tokens = list(_generate_tokens_from_c_tokenizer(
1866+
readline(encoding).__next__,
1867+
extra_tokens=True,
1868+
encoding=encoding,
1869+
))
1870+
self.assertEqual(tokens, expected)
1871+
18361872
def test_int(self):
18371873

18381874
self.check_tokenize('0xff <= 255', """\
@@ -2668,43 +2704,44 @@ def test_unicode(self):
26682704

26692705
def test_invalid_syntax(self):
26702706
def get_tokens(string):
2671-
return list(_generate_tokens_from_c_tokenizer(string))
2672-
2673-
self.assertRaises(SyntaxError, get_tokens, "(1+2]")
2674-
self.assertRaises(SyntaxError, get_tokens, "(1+2}")
2675-
self.assertRaises(SyntaxError, get_tokens, "{1+2]")
2676-
2677-
self.assertRaises(SyntaxError, get_tokens, "1_")
2678-
self.assertRaises(SyntaxError, get_tokens, "1.2_")
2679-
self.assertRaises(SyntaxError, get_tokens, "1e2_")
2680-
self.assertRaises(SyntaxError, get_tokens, "1e+")
2681-
2682-
self.assertRaises(SyntaxError, get_tokens, "\xa0")
2683-
self.assertRaises(SyntaxError, get_tokens, "€")
2684-
2685-
self.assertRaises(SyntaxError, get_tokens, "0b12")
2686-
self.assertRaises(SyntaxError, get_tokens, "0b1_2")
2687-
self.assertRaises(SyntaxError, get_tokens, "0b2")
2688-
self.assertRaises(SyntaxError, get_tokens, "0b1_")
2689-
self.assertRaises(SyntaxError, get_tokens, "0b")
2690-
self.assertRaises(SyntaxError, get_tokens, "0o18")
2691-
self.assertRaises(SyntaxError, get_tokens, "0o1_8")
2692-
self.assertRaises(SyntaxError, get_tokens, "0o8")
2693-
self.assertRaises(SyntaxError, get_tokens, "0o1_")
2694-
self.assertRaises(SyntaxError, get_tokens, "0o")
2695-
self.assertRaises(SyntaxError, get_tokens, "0x1_")
2696-
self.assertRaises(SyntaxError, get_tokens, "0x")
2697-
self.assertRaises(SyntaxError, get_tokens, "1_")
2698-
self.assertRaises(SyntaxError, get_tokens, "012")
2699-
self.assertRaises(SyntaxError, get_tokens, "1.2_")
2700-
self.assertRaises(SyntaxError, get_tokens, "1e2_")
2701-
self.assertRaises(SyntaxError, get_tokens, "1e+")
2702-
2703-
self.assertRaises(SyntaxError, get_tokens, "'sdfsdf")
2704-
self.assertRaises(SyntaxError, get_tokens, "'''sdfsdf''")
2705-
2706-
self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000)
2707-
self.assertRaises(SyntaxError, get_tokens, "]")
2707+
the_string = StringIO(string)
2708+
return list(_generate_tokens_from_c_tokenizer(the_string.readline))
2709+
2710+
for case in [
2711+
"(1+2]",
2712+
"(1+2}",
2713+
"{1+2]",
2714+
"1_",
2715+
"1.2_",
2716+
"1e2_",
2717+
"1e+",
2718+
2719+
"\xa0",
2720+
"€",
2721+
"0b12",
2722+
"0b1_2",
2723+
"0b2",
2724+
"0b1_",
2725+
"0b",
2726+
"0o18",
2727+
"0o1_8",
2728+
"0o8",
2729+
"0o1_",
2730+
"0o",
2731+
"0x1_",
2732+
"0x",
2733+
"1_",
2734+
"012",
2735+
"1.2_",
2736+
"1e2_",
2737+
"1e+",
2738+
"'sdfsdf",
2739+
"'''sdfsdf''",
2740+
"("*1000+"a"+")"*1000,
2741+
"]",
2742+
]:
2743+
with self.subTest(case=case):
2744+
self.assertRaises(SyntaxError, get_tokens, case)
27082745

27092746
def test_max_indent(self):
27102747
MAXINDENT = 100
@@ -2715,20 +2752,24 @@ def generate_source(indents):
27152752
return source
27162753

27172754
valid = generate_source(MAXINDENT - 1)
2718-
tokens = list(_generate_tokens_from_c_tokenizer(valid))
2755+
the_input = StringIO(valid)
2756+
tokens = list(_generate_tokens_from_c_tokenizer(the_input.readline))
27192757
self.assertEqual(tokens[-2].type, DEDENT)
27202758
self.assertEqual(tokens[-1].type, ENDMARKER)
27212759
compile(valid, "<string>", "exec")
27222760

27232761
invalid = generate_source(MAXINDENT)
2724-
self.assertRaises(SyntaxError, lambda: list(_generate_tokens_from_c_tokenizer(invalid)))
2762+
the_input = StringIO(invalid)
2763+
self.assertRaises(SyntaxError, lambda: list(_generate_tokens_from_c_tokenizer(the_input.readline)))
27252764
self.assertRaises(
27262765
IndentationError, compile, invalid, "<string>", "exec"
27272766
)
27282767

27292768
def test_continuation_lines_indentation(self):
27302769
def get_tokens(string):
2731-
return [(kind, string) for (kind, string, *_) in _generate_tokens_from_c_tokenizer(string)]
2770+
the_string = StringIO(string)
2771+
return [(kind, string) for (kind, string, *_)
2772+
in _generate_tokens_from_c_tokenizer(the_string.readline)]
27322773

27332774
code = dedent("""
27342775
def fib(n):

Lib/tokenize.py

+11-21
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import sys
3535
from token import *
3636
from token import EXACT_TOKEN_TYPES
37+
import _tokenize as c_tokenizer
3738

3839
cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
3940
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
@@ -443,29 +444,15 @@ def tokenize(readline):
443444
# BOM will already have been stripped.
444445
encoding = "utf-8"
445446
yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
446-
yield from _tokenize(rl_gen, encoding)
447-
448-
def _tokenize(rl_gen, encoding):
449-
source = b"".join(rl_gen).decode(encoding)
450-
for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
451-
yield token
447+
yield from _generate_tokens_from_c_tokenizer(rl_gen.__next__, encoding, extra_tokens=True)
452448

453449
def generate_tokens(readline):
454450
"""Tokenize a source reading Python code as unicode strings.
455451
456452
This has the same API as tokenize(), except that it expects the *readline*
457453
callable to return str objects instead of bytes.
458454
"""
459-
def _gen():
460-
while True:
461-
try:
462-
line = readline()
463-
except StopIteration:
464-
return
465-
if not line:
466-
return
467-
yield line.encode()
468-
return _tokenize(_gen(), 'utf-8')
455+
return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True)
469456

470457
def main():
471458
import argparse
@@ -502,9 +489,9 @@ def error(message, filename=None, location=None):
502489
tokens = list(tokenize(f.readline))
503490
else:
504491
filename = "<stdin>"
505-
tokens = _tokenize(
492+
tokens = _generate_tokens_from_c_tokenizer(
506493
(x.encode('utf-8') for x in iter(sys.stdin.readline, "")
507-
), "utf-8")
494+
), "utf-8", extra_tokens=True)
508495

509496

510497
# Output the tokenization
@@ -531,10 +518,13 @@ def error(message, filename=None, location=None):
531518
perror("unexpected error: %s" % err)
532519
raise
533520

534-
def _generate_tokens_from_c_tokenizer(source, extra_tokens=False):
521+
def _generate_tokens_from_c_tokenizer(source, encoding=None, extra_tokens=False):
535522
"""Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
536-
import _tokenize as c_tokenizer
537-
for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
523+
if encoding is None:
524+
it = c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens)
525+
else:
526+
it = c_tokenizer.TokenizerIter(source, encoding=encoding, extra_tokens=extra_tokens)
527+
for info in it:
538528
yield TokenInfo._make(info)
539529

540530

0 commit comments

Comments
 (0)