fixup! fixup! fixup! fixup! pythongh-105069: Add a readline-like callable to the tokenizer to consume input iteratively

Author: pablogsal

Lib/test/test_tokenize.py

@@ -64,6 +64,12 @@ def gen():
         with self.assertRaises(TypeError):
             list(generate_tokens(gen().__next__))
 
+        def gen():
+            yield "sdfosdg"
+            1/0
+        with self.assertRaises(ZeroDivisionError):
+            list(generate_tokens(gen().__next__))
+
     def test_implicit_newline(self):
         # Make sure that the tokenizer puts in an implicit NEWLINE
         # when the input lacks a trailing new line.

Parser/tokenizer.c

@@ -140,8 +140,9 @@ static char *
 error_ret(struct tok_state *tok) /* XXX */
 {
     tok->decoding_erred = 1;
-    if (tok->fp != NULL && tok->buf != NULL) /* see _PyTokenizer_Free */
+    if ((tok->fp != NULL || tok->readline != NULL) && tok->buf != NULL) {/* see _PyTokenizer_Free */
         PyMem_Free(tok->buf);
+    }
     tok->buf = tok->cur = tok->inp = NULL;
     tok->start = NULL;
     tok->end = NULL;