gh-153569: Simplify tokenizer state (#158124)

* Use diagnostic line for missing-brace errors

* Keep implicit prepared newline metadata in the reader

* Reuse parsed formatted string kind in lexer

* Transfer owned tokenizer encoding strings

* Remove unused tokenizer state and redundant branches

* Avoid rescanning partial decoded tokenizer lines

* Reuse prepared tokenizer line terminators for CR detection

* Reuse interactive tokenizer chunks without carriage returns

* Release tokenizer constructor resources on allocation failure

* Narrow interactive newline normalization wrapper

* Close duplicated tokenizer descriptors when fdopen fails

* Preserve cached tokenizer lines when newline allocation fails

* Collect cycles involving tokenizer readline callbacks

* Preserve buffered text when decoding source files

* Clear the transferred syntax error line before cleanup

* Propagate tokenizer column conversion failures

* Allocate parser repetition buffers on first match

* Allocate type comment storage only when needed

* Reuse decoded multiline text when counting token columns

* Skip empty decoded tokenizer chunks

* Check parser array growth before allocating

* Name tokenizer bounds and derive byte lengths from their data

* Set the tokenizer exception when file input size overflows

* gh-153569: Send exact line endings in the REPL test

* Share parser repetition buffer growth

* Fold Unicode input coverage into the existing REPL test

* Keep parser array growth policy private

* Drop obsolete bitmap stress from source discard test

* Preserve input failures while scanning string literals

* fixup! Preserve input failures while scanning string literals

* fixup! fixup! Preserve input failures while scanning string literals

* gh-153569: Remove assertion-only lexer bookkeeping

Author: pablogsal

Lib/test/test_fstring.py

@@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self):
                             f"{prefix[-1]}-string: expecting '}}' to close '{{' "
                             f"on line {lineno}")
 
+    def test_unclosed_replacement_field_quote_line(self):
+        for prefix in ('f', 't', 'rf', 'rt'):
+            for quote in ('"', "'"):
+                triple = quote * 3
+                for suffix in ('', '\nx'):
+                    source = prefix + triple + '{1' + triple + suffix
+                    with self.subTest(source=source):
+                        with self.assertRaises(SyntaxError) as cm:
+                            compile(source, '<test>', 'exec')
+                        self.assertEqual(
+                            cm.exception.msg,
+                            f"{prefix[-1]}-string: expecting '}}'")
+                        self.assertEqual(cm.exception.lineno, 1)
+
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     def test_mismatched_parens(self):
         self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' "

Lib/test/test_repl.py

@@ -187,16 +187,19 @@ def read_until(marker, start=0):
     def test_lexer_buffer_realloc_with_null_start(self):
         # gh-144759: NULL pointer arithmetic when the lexer buffer grows
         # while parsing long input.
-        long_value = "a" * 2000
+        long_value = "é漢" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'
         print(x)
         """)
-        p = spawn_repl()
-        p.stdin.write(user_input)
-        output = kill_python(p)
-        self.assertEqual(p.returncode, 0)
-        self.assertIn(long_value, output)
+        for newline in ("\n", "\r\n"):
+            with self.subTest(newline=newline):
+                p = spawn_repl(encoding="utf-8")
+                # Bypass Windows text-mode translation of CRLF to CRCRLF.
+                p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8"))
+                output = kill_python(p)
+                self.assertEqual(p.returncode, 0)
+                self.assertIn(long_value, output)
 
     @cpython_only
     def test_multiline_fstring_source_reallocation(self):

Lib/test/test_source_encoding.py

@@ -29,6 +29,30 @@ def test_compilestring(self):
         exec(c, d)
         self.assertEqual(d['u'], '\xf3')
 
+    def test_compilestring_line_endings(self):
+        for newline in ('\n', '\r', '\r\n', '\r\r\n'):
+            expected = 'é\n\ntext' if newline == '\r\r\n' else 'é\ntext'
+            for suffix in ('', newline):
+                for mode in ('exec', 'eval'):
+                    source = f"'''é{newline}text'''{suffix}"
+                    if mode == 'exec':
+                        source = 'value = ' + source
+                    for encoding in (None, 'utf-8', 'latin-1'):
+                        with self.subTest(newline=newline, suffix=suffix,
+                                          mode=mode, encoding=encoding):
+                            input = source
+                            if encoding is not None:
+                                input = (f'# coding: {encoding}{newline}'
+                                         + source).encode(encoding)
+                            code = compile(input, '<test>', mode)
+                            if mode == 'exec':
+                                namespace = {}
+                                exec(code, namespace)
+                                value = namespace['value']
+                            else:
+                                value = eval(code)
+                            self.assertEqual(value, expected)
+
     def test_issue2301(self):
         try:
             compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
@@ -135,6 +159,11 @@ def test_stateful_file_decoder_spans_lines(self):
         )
         self._assert_python_file_ok(source)
 
+    @support.requires_subprocess()
+    def test_stateful_file_decoder_preserves_buffered_text(self):
+        source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n"
+        self._assert_python_file_ok(source)
+
     @support.requires_subprocess()
     def test_stateful_file_decoder_finalizes_before_implicit_newline(self):
         source = b"# coding: hz\n# ~{1dA?"

Lib/test/test_tokenize.py

@@ -8,6 +8,7 @@
 import token
 import tokenize
 import unittest
+import weakref
 from io import BytesIO, StringIO
 from textwrap import dedent
 from unittest import TestCase, mock
@@ -2268,6 +2269,18 @@ def check_tokenize(self, s, expected):
             )
             self.assertEqual(result, expected.rstrip().splitlines())
 
+    def test_readline_reference_cycle(self):
+        class Readline:
+            def __call__(self):
+                return ""
+
+        readline = Readline()
+        readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
+        ref = weakref.ref(readline)
+        del readline
+        support.gc_collect()
+        self.assertIsNone(ref())
+
     def test_encoding(self):
         def readline(encoding):
             yield "1+1".encode(encoding)
@@ -2341,6 +2354,20 @@ def test_utf8_decoder_spans_readline_calls(self):
             tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""),
         ])
 
+    def test_utf8_decoder_spans_many_readline_calls(self):
+        for prefix in (b"", b"previous\n"):
+            with self.subTest(prefix=prefix):
+                chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100
+                          + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""])
+                source = b"".join(chunks)
+                expected = list(_tokenize.TokenizerIter(
+                    BytesIO(source).readline, encoding="utf-8", extra_tokens=True
+                ))
+                tokens = list(_tokenize.TokenizerIter(
+                    iter(chunks).__next__, encoding="utf-8", extra_tokens=True
+                ))
+                self.assertEqual(tokens, expected)
+
     def test_utf8_decoder_replaces_incomplete_input_at_eof(self):
         expected = [
             tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"),
@@ -2396,6 +2423,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self):
         )
         self.assertEqual(readline.call_count, 2)
 
+    def test_readline_memory_error_in_string(self):
+        readline = mock.Mock(side_effect=['"""first\n', MemoryError])
+        iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
+        with self.assertRaises(MemoryError):
+            next(iterator)
+
     def test_readline_callback_is_not_read_ahead(self):
         readline = mock.Mock(side_effect=["x\n", "y\n", ""])
         iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)

Lib/test/test_type_comments.py

@@ -338,6 +338,16 @@ def test_ignores(self):
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 
+    def test_many_ignores(self):
+        comment_count = 25
+        tags = [f"[tag_{index}]" for index in range(comment_count)]
+        source = "".join(f"pass  # type: ignore{tag}\n" for tag in tags)
+        for tree in self.parse_all(source):
+            self.assertEqual(
+                [(item.lineno, item.tag) for item in tree.type_ignores],
+                list(enumerate(tags, start=1)),
+            )
+
     def test_longargs(self):
         for tree in self.parse_all(longargs, minver=8):
             for t in tree.body:

Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst

@@ -0,0 +1,4 @@
+Correct missing-brace error messages for multiline f-strings and t-strings.
+Improve tokenization of UTF-8 input split across many readline calls.
+Preserve source order when reading files with stateful encodings.
+Preserve input errors when scanning string literals.

Modules/_testinternalcapi/tokenizer.c

@@ -38,6 +38,12 @@ static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
+    const char multiple_lines[] = "a\nb\n";
+    const char first_line[] = "alpha\n";
+    const char second_line[] = "\xce\xb2\n";
+    const char expected[] = "alpha\n\xce\xb2\n";
+    const char tail[] = "tail";
+    const char terminated_line[] = "x\n";
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
@@ -46,23 +52,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     }
 
     if (check_system_error(
-            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+            _PyTok_SourceAppendLine(&source, "", 0) < 0,
             "accepted empty source line") < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, multiple_lines, sizeof(multiple_lines) - 1) < 0,
                 "accepted multiple source lines") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "alpha\n", 6, 0) == 0,
+                      &source, first_line, sizeof(first_line) - 1) == 0,
                   "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "\xce\xb2\n", 3, 1) == 6,
-                  "wrong second source offset") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
-                      _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong implicit newline flags") < 0) {
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      (Py_ssize_t)sizeof(first_line) - 1,
+                  "wrong second source offset") < 0) {
         goto error;
     }
 
@@ -74,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
-    if (check(source.len == 9 &&
-                  memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
+    if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 &&
+                  memcmp(source.bytes, expected, sizeof(expected)) == 0,
               "wrong source contents") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
+    if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, terminated_line, sizeof(terminated_line) - 1) < 0,
                 "appended after unterminated source line") < 0) {
         goto error;
     }
@@ -104,27 +107,35 @@ static PyObject *
 test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
                              PyObject *Py_UNUSED(args))
 {
+    enum { LINE_COUNT = 2 };
+    const char first_line[] = "x\n";
+    const char second_line[] = "y\n";
+    const char tail[] = "tail";
+    const char final_line[] = "z\n";
+    const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1);
+    const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1);
+    const _PyTok_Off discarded_len = first_batch_len + second_batch_len;
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
-    for (int i = 0; i < 260; i++) {
-        if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) {
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) {
             goto error;
         }
     }
     char *bytes = source.bytes;
     _PyTok_Off capacity = source.cap;
     _PyTok_SourceDiscard(&source);
-    if (check(source.base_offset == 520 && source.len == 0 &&
+    if (check(source.base_offset == first_batch_len && source.len == 0 &&
                   source.nlines == 0 && source.bytes == bytes &&
                   source.cap == capacity && source.bytes[0] == '\0',
               "discard did not preserve source allocation") < 0) {
         goto error;
     }
-    for (int i = 0; i < 260; i++) {
-        if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i,
-                  "wrong source offset after discard") < 0 ||
-                check(!_PyTok_SourceLineIsImplicit(&source, i + 1),
-                      "discard preserved implicit newline flag") < 0) {
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (check(_PyTok_SourceAppendLine(
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i,
+                  "wrong source offset after discard") < 0) {
             goto error;
         }
     }
@@ -133,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, tail, sizeof(tail) - 1) == discarded_len,
               "wrong source offset after repeated discard") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) ==
+                  discarded_len + (Py_ssize_t)sizeof(tail) - 1,
               "cannot append after discarding unterminated line") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
     source.base_offset = PY_SSIZE_T_MAX - 1;
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 &&
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) < 0 &&
                   PyErr_ExceptionMatches(PyExc_MemoryError),
               "accepted overflowing logical source offset") < 0) {
         goto error;

Parser/lexer/lexer.c

@@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
 
                 // Handle valid f or t string creation:
                 if (saw_f || saw_t) {
-                    return _PyLexer_scan_fstring_start(tok, token, c);
+                    ftstring_kind kind = saw_t
+                        ? (saw_r ? RAW_TSTRING : TSTRING)
+                        : (saw_r ? RAW_FSTRING : FSTRING);
+                    return _PyLexer_scan_fstring_start(tok, token, c, kind);
                 }
                 return _PyLexer_scan_string(tok, token, c);
             }

Parser/lexer/lexer_internal.h

@@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr(
 void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
-int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
+int _PyLexer_scan_fstring_start(
+    struct tok_state *, struct token *, int, ftstring_kind);
 int _PyLexer_scan_string(struct tok_state *, struct token *, int);
 int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *);
 int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *);