Merge pull request #3921 from Earlopain/revert-3918

Revert "Fix lexing for unterminated strings/heredocs etc."

Author: Earlopain

src/prism.c

@@ -9856,15 +9856,6 @@ parser_lex(pm_parser_t *parser) {
             // We'll check if we're at the end of the file. If we are, then we
             // need to return the EOF token.
             if (parser->current.end >= parser->end) {
-                // We may be missing closing tokens. We should pop modes one by one
-                // to do the appropriate cleanup like moving next_start for heredocs.
-                // Only when no mode is remaining will we actually emit the EOF token.
-                if (parser->lex_modes.current->mode != PM_LEX_DEFAULT) {
-                    lex_mode_pop(parser);
-                    parser_lex(parser);
-                    return;
-                }
-
                 // If we hit EOF, but the EOF came immediately after a newline,
                 // set the start of the token to the newline.  This way any EOF
                 // errors will be reported as happening on that line rather than

test/prism/errors/unterminated_heredoc_and_embexpr.txt

@@ -48,58 +48,11 @@ def test_parse_lex_file
     end
 
     if RUBY_VERSION >= "3.3"
-      def test_lex_compat
-        source = "foo bar"
-        prism = Prism.lex_compat(source, version: "current").value
-        ripper = Ripper.lex(source)
+      def test_lex_compare
+        prism = Prism.lex_compat(File.read(__FILE__), version: "current").value
+        ripper = Ripper.lex(File.read(__FILE__))
         assert_equal(ripper, prism)
       end
     end
-
-    def test_lex_interpolation_unterminated
-      assert_equal(
-        %i[STRING_BEGIN EMBEXPR_BEGIN EOF],
-        token_types('"#{')
-      )
-
-      assert_equal(
-        %i[STRING_BEGIN EMBEXPR_BEGIN IGNORED_NEWLINE EOF],
-        token_types('"#{' + "\n")
-      )
-    end
-
-    def test_lex_interpolation_unterminated_with_content
-      # FIXME: Emits EOL twice.
-      assert_equal(
-        %i[STRING_BEGIN EMBEXPR_BEGIN CONSTANT EOF EOF],
-        token_types('"#{C')
-      )
-
-      assert_equal(
-        %i[STRING_BEGIN EMBEXPR_BEGIN CONSTANT NEWLINE EOF],
-        token_types('"#{C' + "\n")
-      )
-    end
-
-    def test_lex_heredoc_unterminated
-      code = <<~'RUBY'.strip
-        <<A+B
-        #{C
-      RUBY
-
-      assert_equal(
-        %i[HEREDOC_START EMBEXPR_BEGIN CONSTANT HEREDOC_END PLUS CONSTANT NEWLINE EOF],
-        token_types(code)
-      )
-
-      assert_equal(
-        %i[HEREDOC_START EMBEXPR_BEGIN CONSTANT NEWLINE HEREDOC_END PLUS CONSTANT NEWLINE EOF],
-        token_types(code + "\n")
-      )
-    end
-
-    def token_types(code)
-      Prism.lex(code).value.map { |token, _state| token.type }
-    end
   end
 end