-
-
Notifications
You must be signed in to change notification settings - Fork 35.3k
gh-156047: Rank tokens before probing for a keyword typo #156087
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
|
|
||
| import collections.abc | ||
| import functools | ||
| import heapq | ||
| import itertools | ||
| import linecache | ||
| import os | ||
|
|
@@ -19,6 +20,7 @@ | |
|
|
||
| from contextlib import suppress | ||
| lazy import _colorize | ||
| lazy import difflib | ||
|
|
||
| try: | ||
| from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES | ||
|
|
@@ -1427,20 +1429,17 @@ def _find_keyword_typos(self): | |
| if not self._exc_metadata: | ||
| return | ||
|
|
||
| line, offset, source = self._exc_metadata | ||
| line, _, source = self._exc_metadata | ||
| end_line = int(self.lineno) if self.lineno is not None else 0 | ||
| lines = None | ||
| from_filename = False | ||
|
|
||
| if source is None: | ||
| if self.filename: | ||
| try: | ||
| with open(self.filename) as f: | ||
| lines = f.read().splitlines() | ||
| except Exception: | ||
| line, end_line, offset = 0,1,0 | ||
| else: | ||
| from_filename = True | ||
| line, end_line, _ = 0,1,0 | ||
| lines = lines if lines is not None else self.text.splitlines() | ||
| else: | ||
| lines = source.splitlines() | ||
|
|
@@ -1462,26 +1461,32 @@ def _find_keyword_typos(self): | |
| return # Original code compiles or is incomplete - can't validate fixes | ||
|
|
||
| error_lines = error_code.splitlines() | ||
| tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) | ||
| tokens = [] | ||
| offset = self.end_offset | ||
| try: | ||
| for token in tokenize.generate_tokens(io.StringIO(error_code).readline): | ||
| if token.type != tokenize.NAME: | ||
| continue | ||
| if keyword.iskeyword(token.string): | ||
| continue | ||
| # Only consider NAME tokens on the same line as the error | ||
| the_end = end_line if line == 0 else end_line + 1 | ||
| if token.start[0] + line != the_end: | ||
| continue | ||
|
johnslavik marked this conversation as resolved.
|
||
| rank = abs(offset - token.end[1]) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the rank anchor is a bit off here:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'll also add a case where
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This can prefer the token at the parser caret over the actual typo.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Somewhat related fun fact, you can trigger this in main too: def outer():
if True:
pass
retrun a+bThis is because
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Some fix will be needed in
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The other issue I mentioned is tracked in #156092. |
||
| heapq.heappush(tokens, (rank, token)) | ||
| except Exception: | ||
| pass | ||
|
johnslavik marked this conversation as resolved.
|
||
| tokens_left_to_process = 10 | ||
| import difflib | ||
| for token in tokens: | ||
| start, end = token.start, token.end | ||
| if token.type != tokenize.NAME: | ||
| continue | ||
| # Only consider NAME tokens on the same line as the error | ||
| the_end = end_line if line == 0 else end_line + 1 | ||
| if from_filename and token.start[0]+line != the_end: | ||
| continue | ||
| while tokens: | ||
| rank, token = heapq.heappop(tokens) | ||
| wrong_name = token.string | ||
| if wrong_name in keyword.kwlist: | ||
| continue | ||
|
|
||
| # Limit the number of valid tokens to consider to not spend | ||
| # to much time in this function | ||
| tokens_left_to_process -= 1 | ||
| if tokens_left_to_process < 0: | ||
| break | ||
| start, end = token.start, token.end | ||
| # Limit the number of possible matches to try | ||
| max_matches = 3 | ||
| matches = [] | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| When looking for possibly misspelled Python keywords after a :exc:`SyntaxError`, | ||
| candidate names are now ranked to improve accuracy. Patch by Bartosz Sławecki. |
Uh oh!
There was an error while loading. Please reload this page.