gh-153569: streamline tokenizer buffer hot paths

Author: pablogsal

Parser/lexer/lexer.h

@@ -14,10 +14,12 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token,
                   Py_ssize_t *length)
 {
     assert(length != NULL);
-    if (!_PyTok_SpanIsValid(token->span)) {
+    if (token->span.start < 0) {
+        assert(token->span.start == -1 && token->span.end == -1);
         *length = 0;
         return "";
     }
+    assert(_PyTok_SpanIsValid(token->span));
     assert(tok->buf != NULL);
     assert(tok->inp >= tok->buf);
     assert(token->span.start >= tok->buf_offset);

Parser/tokenizer/reader.c

@@ -78,12 +78,11 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
     assert(tok->inp - tok->buf <= reader->input_buffer_cap);
     _PyLexer_BufferPointers pointers;
     _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers);
-    char *buffer = tok->buf;
     if (reserve_buffer(
-            &buffer, &reader->input_buffer_cap, needed) < 0) {
+            &tok->buf, &reader->input_buffer_cap, needed) < 0) {
         return -1;
     }
-    _PyLexer_RestoreBufferPointers(tok, buffer, &pointers);
+    _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers);
     return 0;
 }
 

Python/Python-tokenize.c

@@ -268,7 +268,8 @@ tokenizeriter_next(PyObject *op)
     }
     const char *token_start = NULL;
     PyObject *str;
-    if (!_PyTok_SpanIsValid(token.span)) {
+    if (token.span.start < 0) {
+        assert(token.span.start == -1 && token.span.end == -1);
         str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     }
     else {