gh-156891: Fix shlex source inclusion at EOF

Fix shlex source inclusion when the source filename is the final token of its parent stream. Newly pushed streams now start with a valid lexical state, and popping a stream restores the parent state.

Author: lpyu001

Lib/shlex.py

@@ -77,10 +77,12 @@ def push_source(self, newstream, newfile=None):
         "Push an input source onto the lexer's input source stack."
         if isinstance(newstream, str):
             newstream = StringIO(newstream)
-        self.filestack.appendleft((self.infile, self.instream, self.lineno))
+        self.filestack.appendleft((self.infile, self.instream, self.lineno,
+                                   self.state))
         self.infile = newfile
         self.instream = newstream
         self.lineno = 1
+        self.state = ' '
         if self.debug:
             if newfile is not None:
                 print('shlex: pushing to file %s' % (self.infile,))
@@ -90,11 +92,11 @@ def push_source(self, newstream, newfile=None):
     def pop_source(self):
         "Pop the input source stack."
         self.instream.close()
-        (self.infile, self.instream, self.lineno) = self.filestack.popleft()
+        (self.infile, self.instream, self.lineno,
+         self.state) = self.filestack.popleft()
         if self.debug:
             print('shlex: popping to %s, line %d' \
                   % (self.instream, self.lineno))
-        self.state = ' '
 
     def get_token(self):
         "Get a token from the input stream (or from stack if it's nonempty)"

Lib/test/test_shlex.py

@@ -420,6 +420,24 @@ def testPushSourceStream(self):
         s.push_source(io.StringIO("hello"))
         self.assertListEqual(list(s), ["hello", "world"])
 
+    def testPushSourceKeepsPushback(self):
+        s = shlex.shlex("parent")
+        stream = io.StringIO("child")
+        s.push_token("pushed")
+        s.push_source(stream)
+        self.assertListEqual(list(s), ["pushed", "child", "parent"])
+        self.assertTrue(stream.closed)
+
+    def testPushSourceAfterEOF(self):
+        for posix in (False, True):
+            with self.subTest(posix=posix):
+                s = shlex.shlex("parent", posix=posix)
+                self.assertEqual(list(s), ["parent"])
+                stream = io.StringIO("child")
+                s.push_source(stream)
+                self.assertEqual(list(s), ["child"])
+                self.assertTrue(stream.closed)
+
     def testPushSourceStreamDebug(self):
         s = shlex.shlex("")
         stream = io.StringIO("hello")
@@ -514,6 +532,29 @@ def testSourceInclusion(self):
         s.sourcehook = lambda f: (f, io.StringIO("included"))
         self.assertEqual(list(s), ["included", "remaining"])
 
+    def testSourceInclusionAtEOF(self):
+        for posix in (False, True):
+            with self.subTest(posix=posix):
+                s = shlex.shlex("trigger filename", posix=posix)
+                s.source = "trigger"
+                stream = io.StringIO("included")
+                s.sourcehook = lambda f: (f, stream)
+                self.assertEqual(list(s), ["included"])
+                self.assertTrue(stream.closed)
+
+    def testNestedSourceInclusionAtEOF(self):
+        for posix in (False, True):
+            with self.subTest(posix=posix):
+                streams = {
+                    "child": io.StringIO("trigger grandchild"),
+                    "grandchild": io.StringIO("included"),
+                }
+                s = shlex.shlex("trigger child", posix=posix)
+                s.source = "trigger"
+                s.sourcehook = lambda f: (f, streams[f])
+                self.assertEqual(list(s), ["included"])
+                self.assertTrue(all(stream.closed for stream in streams.values()))
+
     def testGetTokenPopsPushbackDebug(self):
         s = shlex.shlex("")
         s.push_token("hello")

Misc/NEWS.d/next/Library/2026-09-03-20-54-42.gh-issue-156891.fliROC.rst

@@ -0,0 +1,2 @@
+Fix :class:`shlex.shlex` source inclusion when the source filename is the
+final token in a stream.