1
1
from test import support
2
2
from test .support import os_helper
3
- from tokenize import (tokenize , _tokenize , untokenize , NUMBER , NAME , OP ,
3
+ from tokenize import (tokenize , untokenize , NUMBER , NAME , OP ,
4
4
STRING , ENDMARKER , ENCODING , tok_name , detect_encoding ,
5
5
open as tokenize_open , Untokenizer , generate_tokens ,
6
6
NEWLINE , _generate_tokens_from_c_tokenizer , DEDENT , TokenInfo )
@@ -50,7 +50,20 @@ def check_tokenize(self, s, expected):
50
50
self .assertEqual (result ,
51
51
[" ENCODING 'utf-8' (0, 0) (0, 0)" ] +
52
52
expected .rstrip ().splitlines ())
53
-
53
+
54
+ def test_invalid_readline (self ):
55
+ def gen ():
56
+ yield "sdfosdg"
57
+ yield "sdfosdg"
58
+ with self .assertRaises (TypeError ):
59
+ list (tokenize (gen ().__next__ ))
60
+
61
+ def gen ():
62
+ yield b"sdfosdg"
63
+ yield b"sdfosdg"
64
+ with self .assertRaises (TypeError ):
65
+ list (generate_tokens (gen ().__next__ ))
66
+
54
67
def test_implicit_newline (self ):
55
68
# Make sure that the tokenizer puts in an implicit NEWLINE
56
69
# when the input lacks a trailing new line.
@@ -1154,7 +1167,8 @@ class TestTokenizerAdheresToPep0263(TestCase):
1154
1167
1155
1168
def _testFile (self , filename ):
1156
1169
path = os .path .join (os .path .dirname (__file__ ), filename )
1157
- TestRoundtrip .check_roundtrip (self , open (path , 'rb' ))
1170
+ with open (path , 'rb' ) as f :
1171
+ TestRoundtrip .check_roundtrip (self , f )
1158
1172
1159
1173
def test_utf8_coding_cookie_and_no_utf8_bom (self ):
1160
1174
f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
@@ -1199,7 +1213,8 @@ def readline():
1199
1213
yield b''
1200
1214
1201
1215
# skip the initial encoding token and the end tokens
1202
- tokens = list (_tokenize (readline (), encoding = 'utf-8' ))[:- 2 ]
1216
+ tokens = list (_generate_tokens_from_c_tokenizer (readline ().__next__ , encoding = 'utf-8' ,
1217
+ extra_tokens = True ))[:- 2 ]
1203
1218
expected_tokens = [TokenInfo (3 , '"ЉЊЈЁЂ"' , (1 , 0 ), (1 , 7 ), '"ЉЊЈЁЂ"\n ' )]
1204
1219
self .assertEqual (tokens , expected_tokens ,
1205
1220
"bytes not decoded with encoding" )
@@ -1468,13 +1483,13 @@ def test_tokenize(self):
1468
1483
def mock_detect_encoding (readline ):
1469
1484
return encoding , [b'first' , b'second' ]
1470
1485
1471
- def mock__tokenize (readline , encoding ):
1486
+ def mock__tokenize (readline , encoding , ** kwargs ):
1472
1487
nonlocal encoding_used
1473
1488
encoding_used = encoding
1474
1489
out = []
1475
1490
while True :
1476
1491
try :
1477
- next_line = next ( readline )
1492
+ next_line = readline ( )
1478
1493
except StopIteration :
1479
1494
return out
1480
1495
if next_line :
@@ -1491,16 +1506,16 @@ def mock_readline():
1491
1506
return str (counter ).encode ()
1492
1507
1493
1508
orig_detect_encoding = tokenize_module .detect_encoding
1494
- orig__tokenize = tokenize_module ._tokenize
1509
+ orig_c_token = tokenize_module ._generate_tokens_from_c_tokenizer
1495
1510
tokenize_module .detect_encoding = mock_detect_encoding
1496
- tokenize_module ._tokenize = mock__tokenize
1511
+ tokenize_module ._generate_tokens_from_c_tokenizer = mock__tokenize
1497
1512
try :
1498
1513
results = tokenize (mock_readline )
1499
1514
self .assertEqual (list (results )[1 :],
1500
1515
[b'first' , b'second' , b'1' , b'2' , b'3' , b'4' ])
1501
1516
finally :
1502
1517
tokenize_module .detect_encoding = orig_detect_encoding
1503
- tokenize_module ._tokenize = orig__tokenize
1518
+ tokenize_module ._generate_tokens_from_c_tokenizer = orig_c_token
1504
1519
1505
1520
self .assertEqual (encoding_used , encoding )
1506
1521
@@ -1827,12 +1842,33 @@ class CTokenizeTest(TestCase):
1827
1842
def check_tokenize (self , s , expected ):
1828
1843
# Format the tokens in s in a table format.
1829
1844
# The ENDMARKER and final NEWLINE are omitted.
1845
+ f = StringIO (s )
1830
1846
with self .subTest (source = s ):
1831
1847
result = stringify_tokens_from_source (
1832
- _generate_tokens_from_c_tokenizer (s ), s
1848
+ _generate_tokens_from_c_tokenizer (f . readline ), s
1833
1849
)
1834
1850
self .assertEqual (result , expected .rstrip ().splitlines ())
1835
1851
1852
+ def test_encoding (self ):
1853
+ def readline (encoding ):
1854
+ yield "1+1" .encode (encoding )
1855
+
1856
+ expected = [
1857
+ TokenInfo (type = NUMBER , string = '1' , start = (1 , 0 ), end = (1 , 1 ), line = '1+1\n ' ),
1858
+ TokenInfo (type = OP , string = '+' , start = (1 , 1 ), end = (1 , 2 ), line = '1+1\n ' ),
1859
+ TokenInfo (type = NUMBER , string = '1' , start = (1 , 2 ), end = (1 , 3 ), line = '1+1\n ' ),
1860
+ TokenInfo (type = NEWLINE , string = '\n ' , start = (1 , 3 ), end = (1 , 4 ), line = '1+1\n ' ),
1861
+ TokenInfo (type = ENDMARKER , string = '' , start = (2 , 0 ), end = (2 , 0 ), line = '' )
1862
+ ]
1863
+ for encoding in ["utf-8" , "latin-1" , "utf-16" ]:
1864
+ with self .subTest (encoding = encoding ):
1865
+ tokens = list (_generate_tokens_from_c_tokenizer (
1866
+ readline (encoding ).__next__ ,
1867
+ extra_tokens = True ,
1868
+ encoding = encoding ,
1869
+ ))
1870
+ self .assertEqual (tokens , expected )
1871
+
1836
1872
def test_int (self ):
1837
1873
1838
1874
self .check_tokenize ('0xff <= 255' , """\
@@ -2668,43 +2704,44 @@ def test_unicode(self):
2668
2704
2669
2705
def test_invalid_syntax (self ):
2670
2706
def get_tokens (string ):
2671
- return list (_generate_tokens_from_c_tokenizer (string ))
2672
-
2673
- self .assertRaises (SyntaxError , get_tokens , "(1+2]" )
2674
- self .assertRaises (SyntaxError , get_tokens , "(1+2}" )
2675
- self .assertRaises (SyntaxError , get_tokens , "{1+2]" )
2676
-
2677
- self .assertRaises (SyntaxError , get_tokens , "1_" )
2678
- self .assertRaises (SyntaxError , get_tokens , "1.2_" )
2679
- self .assertRaises (SyntaxError , get_tokens , "1e2_" )
2680
- self .assertRaises (SyntaxError , get_tokens , "1e+" )
2681
-
2682
- self .assertRaises (SyntaxError , get_tokens , "\xa0 " )
2683
- self .assertRaises (SyntaxError , get_tokens , "€" )
2684
-
2685
- self .assertRaises (SyntaxError , get_tokens , "0b12" )
2686
- self .assertRaises (SyntaxError , get_tokens , "0b1_2" )
2687
- self .assertRaises (SyntaxError , get_tokens , "0b2" )
2688
- self .assertRaises (SyntaxError , get_tokens , "0b1_" )
2689
- self .assertRaises (SyntaxError , get_tokens , "0b" )
2690
- self .assertRaises (SyntaxError , get_tokens , "0o18" )
2691
- self .assertRaises (SyntaxError , get_tokens , "0o1_8" )
2692
- self .assertRaises (SyntaxError , get_tokens , "0o8" )
2693
- self .assertRaises (SyntaxError , get_tokens , "0o1_" )
2694
- self .assertRaises (SyntaxError , get_tokens , "0o" )
2695
- self .assertRaises (SyntaxError , get_tokens , "0x1_" )
2696
- self .assertRaises (SyntaxError , get_tokens , "0x" )
2697
- self .assertRaises (SyntaxError , get_tokens , "1_" )
2698
- self .assertRaises (SyntaxError , get_tokens , "012" )
2699
- self .assertRaises (SyntaxError , get_tokens , "1.2_" )
2700
- self .assertRaises (SyntaxError , get_tokens , "1e2_" )
2701
- self .assertRaises (SyntaxError , get_tokens , "1e+" )
2702
-
2703
- self .assertRaises (SyntaxError , get_tokens , "'sdfsdf" )
2704
- self .assertRaises (SyntaxError , get_tokens , "'''sdfsdf''" )
2705
-
2706
- self .assertRaises (SyntaxError , get_tokens , "(" * 1000 + "a" + ")" * 1000 )
2707
- self .assertRaises (SyntaxError , get_tokens , "]" )
2707
+ the_string = StringIO (string )
2708
+ return list (_generate_tokens_from_c_tokenizer (the_string .readline ))
2709
+
2710
+ for case in [
2711
+ "(1+2]" ,
2712
+ "(1+2}" ,
2713
+ "{1+2]" ,
2714
+ "1_" ,
2715
+ "1.2_" ,
2716
+ "1e2_" ,
2717
+ "1e+" ,
2718
+
2719
+ "\xa0 " ,
2720
+ "€" ,
2721
+ "0b12" ,
2722
+ "0b1_2" ,
2723
+ "0b2" ,
2724
+ "0b1_" ,
2725
+ "0b" ,
2726
+ "0o18" ,
2727
+ "0o1_8" ,
2728
+ "0o8" ,
2729
+ "0o1_" ,
2730
+ "0o" ,
2731
+ "0x1_" ,
2732
+ "0x" ,
2733
+ "1_" ,
2734
+ "012" ,
2735
+ "1.2_" ,
2736
+ "1e2_" ,
2737
+ "1e+" ,
2738
+ "'sdfsdf" ,
2739
+ "'''sdfsdf''" ,
2740
+ "(" * 1000 + "a" + ")" * 1000 ,
2741
+ "]" ,
2742
+ ]:
2743
+ with self .subTest (case = case ):
2744
+ self .assertRaises (SyntaxError , get_tokens , case )
2708
2745
2709
2746
def test_max_indent (self ):
2710
2747
MAXINDENT = 100
@@ -2715,20 +2752,24 @@ def generate_source(indents):
2715
2752
return source
2716
2753
2717
2754
valid = generate_source (MAXINDENT - 1 )
2718
- tokens = list (_generate_tokens_from_c_tokenizer (valid ))
2755
+ the_input = StringIO (valid )
2756
+ tokens = list (_generate_tokens_from_c_tokenizer (the_input .readline ))
2719
2757
self .assertEqual (tokens [- 2 ].type , DEDENT )
2720
2758
self .assertEqual (tokens [- 1 ].type , ENDMARKER )
2721
2759
compile (valid , "<string>" , "exec" )
2722
2760
2723
2761
invalid = generate_source (MAXINDENT )
2724
- self .assertRaises (SyntaxError , lambda : list (_generate_tokens_from_c_tokenizer (invalid )))
2762
+ the_input = StringIO (invalid )
2763
+ self .assertRaises (SyntaxError , lambda : list (_generate_tokens_from_c_tokenizer (the_input .readline )))
2725
2764
self .assertRaises (
2726
2765
IndentationError , compile , invalid , "<string>" , "exec"
2727
2766
)
2728
2767
2729
2768
def test_continuation_lines_indentation (self ):
2730
2769
def get_tokens (string ):
2731
- return [(kind , string ) for (kind , string , * _ ) in _generate_tokens_from_c_tokenizer (string )]
2770
+ the_string = StringIO (string )
2771
+ return [(kind , string ) for (kind , string , * _ )
2772
+ in _generate_tokens_from_c_tokenizer (the_string .readline )]
2732
2773
2733
2774
code = dedent ("""
2734
2775
def fib(n):
0 commit comments