diff --git a/CHANGELOG.md b/CHANGELOG.md index cb65991657..d3ebfd0cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.61 (unreleased) +- Performance: nested ignore rules are now scoped to the active directory branch during detection, so large monorepos no longer rescan every sibling project's patterns (#2834). - Fix: `graphify.serve` now imports cleanly on Python 3.12 and 3.13. The `chinese` extra pins `jieba-py` from 3.12 onward (0.9.60 mistakenly kept the old `jieba` until 3.14, and its invalid regex escapes are a hard error on 3.12+), and the jieba import now suppresses the tokenizer's `SyntaxWarning` regardless of message or line so it never escalates under `-W error`. - Fix: the git hook's rebuild-root guard now rejects a symlink-loop or dangling `.graphify_root` on Python 3.13, whose `Path.resolve()` no longer raises on a loop — the saved root must resolve to a real directory inside the repo before it is adopted. diff --git a/graphify/detect.py b/graphify/detect.py index 287b0f431a..52c9e56007 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -1794,12 +1794,94 @@ def _wc(path: Path) -> int: ignore_patterns.append((root, line)) explicit_ignore_patterns.append((root, line)) - def _ignored_for_scan(path: Path) -> bool: + # Keep the initial ancestor chain (including CLI excludes) immutable. The + # live walk below only needs rules from the current root-to-directory path; + # retaining rules from completed sibling subtrees makes every later ignore + # check scan an ever-growing list (#2834). + initial_ignore_patterns = tuple(ignore_patterns) + initial_explicit_patterns = tuple(explicit_ignore_patterns) + nested_patterns: dict[ + Path, tuple[list[tuple[Path, str]], list[tuple[Path, str]]] + ] = {} + loaded_pattern_count = len(ignore_patterns) + active_scopes: list[tuple[Path, int, int]] = [(root, 0, 0)] + + def _activate_walk_directory(dp: Path) -> None: + """Make ignore rules match the currently yielded ``os.walk`` branch.""" + nonlocal loaded_pattern_count + if dp == root: + return + + # os.walk(topdown=True) yields siblings after their previous subtree, + # so discard the trailing rows contributed by directories no longer in + # the current path. The root/ancestor rows loaded before the walk stay + # at the front of both lists. + while len(active_scopes) > 1: + current = active_scopes[-1][0] + try: + dp.relative_to(current) + except ValueError: + _, ignore_count, explicit_count = active_scopes.pop() + if ignore_count: + del ignore_patterns[-ignore_count:] + if explicit_count: + del explicit_ignore_patterns[-explicit_count:] + continue + break + + own_patterns = _load_dir_own_ignore(dp, gitignore=gitignore) + own_explicit_patterns = _load_dir_own_ignore(dp, gitignore=False) + nested_patterns[dp] = (own_patterns, own_explicit_patterns) + ignore_patterns.extend(own_patterns) + explicit_ignore_patterns.extend(own_explicit_patterns) + loaded_pattern_count += len(own_patterns) + active_scopes.append( + (dp, len(own_patterns), len(own_explicit_patterns)) + ) + + # ``detect()`` checks directories while walking and files afterwards. The + # latter need the same ancestor chain even though os.walk has moved on, so + # memoize the reconstructed chain by containing directory. + pattern_chain_cache: dict[ + Path, tuple[list[tuple[Path, str]], list[tuple[Path, str]]] + ] = { + root: (list(initial_ignore_patterns), list(initial_explicit_patterns)) + } + + def _patterns_for_directory( + directory: Path, + ) -> tuple[list[tuple[Path, str]], list[tuple[Path, str]]]: + cached = pattern_chain_cache.get(directory) + if cached is not None: + return cached + try: + directory.relative_to(root) + except ValueError: + return pattern_chain_cache[root] + parent_patterns, parent_explicit_patterns = _patterns_for_directory( + directory.parent + ) + own_patterns, own_explicit_patterns = nested_patterns.get( + directory, ([], []) + ) + cached = ( + [*parent_patterns, *own_patterns], + [*parent_explicit_patterns, *own_explicit_patterns], + ) + pattern_chain_cache[directory] = cached + return cached + + def _ignored_for_scan( + path: Path, + *, + patterns: list[tuple[Path, str]] | None = None, + explicit_patterns: list[tuple[Path, str]] | None = None, + ) -> bool: return _is_scan_ignored( path, root, - ignore_patterns, - explicit_ignore_patterns, + ignore_patterns if patterns is None else patterns, + explicit_ignore_patterns if explicit_patterns is None else explicit_patterns, tracked_files, tracked_dirs, cache=ignore_cache, @@ -1853,10 +1935,7 @@ def _on_walk_error(err: OSError) -> None: # Load it now, before pruning dp's children, so a nested ignore # file governs its own subtree the same way git honors it (#1206). if dp != root: - ignore_patterns.extend(_load_dir_own_ignore(dp, gitignore=gitignore)) - explicit_ignore_patterns.extend( - _load_dir_own_ignore(dp, gitignore=False) - ) + _activate_walk_directory(dp) # Prune noise dirs in-place so os.walk never descends into them. # Dot dirs are allowed — users often want .github/, .claude/, etc. # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir. @@ -1914,6 +1993,12 @@ def _on_walk_error(err: OSError) -> None: all_files.sort(key=lambda p: str(p)) + # Results cached during directory pruning were computed against the active + # branch at that point. Rebuild the cache for the per-file pass, whose + # pattern list is reconstructed independently for each containing directory. + ignore_cache.clear() + explicit_ignore_cache.clear() + out_base = Path(cache_root).resolve() if cache_root is not None else root converted_dir = out_base / GRAPHIFY_OUT / "converted" @@ -1924,9 +2009,15 @@ def _on_walk_error(err: OSError) -> None: # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): continue - if not in_memory and _ignored_for_scan(p): - ignored.append(str(p)) - continue + if not in_memory: + file_patterns, file_explicit_patterns = _patterns_for_directory(p.parent) + if _ignored_for_scan( + p, + patterns=file_patterns, + explicit_patterns=file_explicit_patterns, + ): + ignored.append(str(p)) + continue if not _resolves_under_root(p, root): skipped_sensitive.append(str(p) + " [symlink target outside scan root]") continue @@ -2033,7 +2124,7 @@ def _on_walk_error(err: OSError) -> None: "walk_errors": walk_errors, "ignored": sorted(ignored), "pruned_noise_dirs": sorted(pruned_noise), - "graphifyignore_patterns": len(ignore_patterns), + "graphifyignore_patterns": loaded_pattern_count, "scan_root": str(root.resolve()), } diff --git a/tests/test_detect.py b/tests/test_detect.py index 3a26357694..0bb054451b 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -3113,6 +3113,47 @@ def test_nested_gitignore_does_not_govern_sibling_project(tmp_path): ), f"ignored subtree should be recorded in detect()['ignored']: {result['ignored']}" +# --------------------------------------------------------------------------- +# #2834: nested ignore rules must be scoped to the active walk branch. + + +def test_nested_ignore_patterns_are_scoped_to_live_walk_branch(tmp_path, monkeypatch): + """Rules from completed sibling subtrees must not stay active (#2834).""" + import graphify.detect as det + + sibling_count = 12 + (tmp_path / ".graphifyignore").write_text("*.root-noise\n") + (tmp_path / "root.py").write_text("x = 1\n") + for index in range(sibling_count): + project = tmp_path / f"project_{index:02d}" + nested = project / "nested" + nested.mkdir(parents=True) + (project / ".graphifyignore").write_text("*.project-noise\n") + (nested / ".graphifyignore").write_text("*.nested-noise\n") + (project / "keep.py").write_text("x = 1\n") + (project / "drop.project-noise").write_text("x = 1\n") + (nested / "keep.py").write_text("x = 1\n") + (nested / "drop.nested-noise").write_text("x = 1\n") + + active_pattern_counts: list[int] = [] + real_scan = det._is_scan_ignored + + def tracking_scan(path, root, patterns, explicit_patterns, *args, **kwargs): + active_pattern_counts.append(len(patterns)) + return real_scan( + path, root, patterns, explicit_patterns, *args, **kwargs + ) + + monkeypatch.setattr(det, "_is_scan_ignored", tracking_scan) + result = det.detect(tmp_path, gitignore=False) + + assert result["total_files"] == 1 + 2 * sibling_count + assert result["graphifyignore_patterns"] == 1 + 2 * sibling_count + # Root + one project + one nested directory. A sibling's rules must not + # increase the active list after os.walk backtracks to the next project. + assert max(active_pattern_counts) <= 3 + + # --------------------------------------------------------------------------- # #1908: manifest must not retain scan-excluded files as permanent # "deleted" entries. Full-scan saves prune excluded-but-alive rows; subset