diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 548b40f22..3d8d53733 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,40 @@ jobs: uv run --frozen graphify --help uv run --frozen graphify install + platform-paths: + # Keep the filesystem boundary honest on every supported desktop OS. The + # Windows run creates and extracts a real 300+ character source path; the + # same tests verify that the adapter remains a no-op on Linux and macOS. + name: Filesystem paths (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --frozen + + - name: Exercise logical and extended path boundaries + run: >- + uv run --frozen pytest + tests/test_windows_long_paths.py + tests/test_long_path_hashing.py + tests/test_paths.py + tests/test_atomic_writes.py + tests/test_google_workspace.py + tests/test_cargo_introspect.py + tests/test_cpp_preprocess.py + -q --tb=short + security-scan: # The dev deps include bandit and pip-audit. Run them in CI so a new # HIGH-severity finding or vulnerable dependency is caught on the PR that diff --git a/CHANGELOG.md b/CHANGELOG.md index 791142ef2..5a49c36e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## Unreleased + +- Fix: Windows scans no longer skip deeply nested local or UNC files when their paths + cross the legacy `MAX_PATH` boundary. Filesystem calls now use extended-length paths + only at the I/O boundary while graph IDs, manifests, cache keys, and diagnostics retain + ordinary paths; discovery, hashing, extraction, incremental rebuilds, and document/media + readers share the same cross-platform adapter. A focused Ubuntu/macOS/Windows CI matrix + exercises 300+ character paths and verifies that the adapter remains a no-op off Windows. ## 0.9.32 (unreleased) - Fix: incremental extraction and `_rebuild_code` no longer drop a file's other tier (#2333, #2334, #2336). Node/edge ownership was keyed on `source_file` alone, so a semantic re-extract deleted a doc's AST headings and a full rebuild deleted document AST nodes. Merge is now tier-aware (an AST re-extract replaces only AST nodes and keeps the semantic layer, and vice versa), the `_origin` provenance marker is backfilled on load so old graphs self-heal, and the full-rebuild drop is scoped to sources actually regenerated. diff --git a/graphify/build.py b/graphify/build.py index a08219792..0394f9439 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -30,7 +30,12 @@ from pathlib import Path import networkx as nx from .ids import make_id, normalize_id as _normalize_id -from .paths import default_graph_json as _default_graph_json +from .paths import ( + default_graph_json as _default_graph_json, + path_exists as _path_exists, + read_text as _read_text, + resolve_path as _resolve_path, +) from .validate import validate_extraction @@ -251,7 +256,7 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: # matching. Only the slow path resolves, so the common lexical match # stays filesystem-free. try: - p = Path(p).resolve().relative_to(Path(root).resolve()).as_posix() + p = _resolve_path(p).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): pass return p @@ -275,7 +280,7 @@ def _abs_identity(p: str | None, root: str | None = None) -> str | None: if not pp.is_absolute() and root: pp = Path(root) / q try: - return pp.resolve().as_posix() + return _resolve_path(pp).as_posix() except OSError: return pp.as_posix() @@ -365,14 +370,14 @@ def _infer_merge_root(graph_path: Path) -> str | None: """ try: marker = graph_path.parent / ".graphify_root" - if marker.exists(): - recorded = marker.read_text(encoding="utf-8").strip() + if _path_exists(marker): + recorded = _read_text(marker, encoding="utf-8").strip() if recorded: - return str(Path(recorded).resolve()) + return str(_resolve_path(recorded)) except OSError: pass try: - return str(graph_path.parent.parent.resolve()) + return str(_resolve_path(graph_path.parent.parent)) except Exception: return None @@ -617,7 +622,7 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat root: if given, absolute source_file paths from semantic subagents are made relative to root so all nodes share a consistent path key (#932). """ - _root = str(Path(root).resolve()) if root else None + _root = str(_resolve_path(root)) if root else None # NetworkX <= 3.1 serialised edges as "links"; remap to "edges" for compatibility. if "edges" not in extraction and "links" in extraction: extraction = dict(extraction, edges=extraction["links"]) @@ -1228,12 +1233,12 @@ def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list, bool] | N exists but cannot be parsed — callers must refuse to overwrite rather than silently replace a possibly-recoverable graph. """ - if not graph_path.exists(): + if not _path_exists(graph_path): return None from graphify.security import check_graph_file_size_cap check_graph_file_size_cap(graph_path) try: - data = json.loads(graph_path.read_text(encoding="utf-8")) + data = json.loads(_read_text(graph_path, encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: raise RuntimeError( f"Cannot read {graph_path} for incremental merge: {exc}. " @@ -1301,7 +1306,7 @@ def merge_raw_extraction( existing_nodes, existing_edges, existing_hyperedges, _ = loaded _eff_root = ( - str(Path(root).resolve()) if root is not None + str(_resolve_path(root)) if root is not None else _infer_merge_root(graph_path) ) @@ -1420,7 +1425,7 @@ def build_merge( # absolute deleted-file paths never matched the relative node keys and their # nodes survived as ghosts). _eff_root = ( - str(Path(root).resolve()) if root is not None + str(_resolve_path(root)) if root is not None else _infer_merge_root(graph_path) ) @@ -1575,7 +1580,7 @@ def _prune_match(sf: "str | None") -> bool: # Safety check: refuse to shrink the graph silently (#479) # Skip when dedup or prune_sources is active — shrinkage is intentional there. - if graph_path.exists() and not dedup and not prune_sources: + if _path_exists(graph_path) and not dedup and not prune_sources: existing_n = len(existing_nodes) new_n = G.number_of_nodes() if new_n < existing_n: diff --git a/graphify/cache.py b/graphify/cache.py index 354f5213c..0c268b736 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -15,7 +15,22 @@ # shared-output setups. Accepts a relative name ("graphify-out-feature") or an # absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths # (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + glob_paths as _glob_paths, + io_path as _io_path, + iterdir_path as _iterdir_path, + logical_path as _logical_path, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_bytes, + read_text as _read_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, +) # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file @@ -47,18 +62,18 @@ def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: if key in _cleaned_ast_dirs: return _cleaned_ast_dirs.add(key) - if not ast_base.is_dir(): + if not _path_is_dir(ast_base): return import shutil - for child in ast_base.iterdir(): + for child in _iterdir_path(ast_base): if child == current_dir: continue try: - if child.is_dir() and child.name.startswith("v"): - shutil.rmtree(child, ignore_errors=True) + if _path_is_dir(child) and child.name.startswith("v"): + shutil.rmtree(_io_path(child), ignore_errors=True) elif child.suffix == ".json": - child.unlink() + _unlink_path(child) except OSError: pass @@ -100,7 +115,7 @@ def prompt_fingerprint(prompt: "str | Path") -> str: like a prompt change and re-bill extraction. """ if isinstance(prompt, Path): - text = prompt.read_text(encoding="utf-8", errors="replace") + text = _read_text(prompt, encoding="utf-8", errors="replace") else: text = prompt normalized = "\n".join( @@ -131,7 +146,7 @@ class #1939 is about, so the two are not inferred from each other. if prompt_file is not None: prompt = Path(prompt_file) try: - st = prompt.stat() + st = _path_stat(prompt) memo_key = (str(prompt), st.st_size, st.st_mtime_ns) if memo_key in _prompt_fp_cache: return _prompt_fp_cache[memo_key] @@ -231,7 +246,7 @@ def _stat_key_to_absolute(key: str, anchor: Path) -> str: def _stat_index_file(root: Path) -> Path: _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out return base / "cache" / "stat-index.json" @@ -246,13 +261,13 @@ def _ensure_stat_index(root: Path, cache_root: "Path | None" = None) -> None: # in-memory keys stay absolute, but the on-disk index stores in-anchor keys # relative so a moved/cloned corpus still hits (#2199) — same load/save # re-anchoring the detect manifest uses. - _stat_index_root = Path(cache_root if cache_root is not None else root).resolve() - _stat_index_anchor = Path(root).resolve() + _stat_index_root = _resolve_path(cache_root if cache_root is not None else root) + _stat_index_anchor = _resolve_path(root) p = _stat_index_file(_stat_index_root) _stat_index = {} - if p.exists(): + if _path_exists(p): try: - raw = json.loads(p.read_text(encoding="utf-8")) + raw = json.loads(_read_text(p, encoding="utf-8")) if isinstance(raw, dict): for k, v in raw.items(): if not isinstance(k, str): @@ -283,19 +298,23 @@ def _flush_stat_index() -> None: on_disk: dict[str, dict] = {} for k, v in _stat_index.items(): try: - if not os.path.exists(k): + if not _path_exists(k): continue except OSError: continue dk = _stat_key_to_relative(k, _stat_index_anchor) if _stat_index_anchor is not None else k on_disk[dk] = v try: - p.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") + _make_dirs(p.parent, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=_io_path(p.parent), + prefix="stat-index.", + suffix=".tmp", + ) try: os.write(fd, json.dumps(on_disk, separators=(",", ":")).encode()) os.close(fd) - os.replace(tmp, p) + os.replace(tmp, _io_path(p)) except Exception: try: os.close(fd) @@ -315,9 +334,7 @@ def _normalize_path(path: Path) -> Path: import sys if sys.platform != "win32": return path - s = str(path) - if s.startswith("\\\\?\\"): - s = s[4:] # strip extended-length prefix \\?\ + s = _logical_path(path) return Path(os.path.normcase(s)) @@ -338,7 +355,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No global _stat_index_dirty p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) - if not p.is_file(): + if not _path_is_file(p): raise IsADirectoryError(f"file_hash requires a file, got: {p}") # The stat index is a cache artifact, so it must follow the cache location @@ -346,7 +363,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # graphify-out/cache/stat-index.json inside the analyzed source tree even when # the AST cache itself is redirected to CWD (#1774 completion). _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() + resolved = _resolve_path(p) abs_key = str(resolved) # The salt is the path component that enters the digest (relative to root, or # the absolute-path fallback). The stat-index memo MUST be keyed by it too: @@ -356,13 +373,13 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # and poisoning the persisted stat-index across runs (#1989). Store one digest # per salt so alternating roots don't force re-reads. try: - salt = resolved.relative_to(Path(root).resolve()).as_posix().lower() + salt = resolved.relative_to(_resolve_path(root)).as_posix().lower() except ValueError: salt = resolved.as_posix().lower() st: "os.stat_result | None" = None try: - st = p.stat() + st = _path_stat(p) entry = _stat_index.get(abs_key) if (isinstance(entry, dict) and entry.get("size") == st.st_size @@ -377,7 +394,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No except OSError: pass - raw = p.read_bytes() + raw = _read_bytes(p) content = _body_content(raw) if p.suffix.lower() == ".md" else raw h = hashlib.sha256() h.update(content) @@ -421,10 +438,10 @@ def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) + abs_key = str(_resolve_path(p)) st: "os.stat_result | None" = None try: - st = p.stat() + st = _path_stat(p) entry = _stat_index.get(abs_key) if (entry and entry.get("size") == st.st_size @@ -465,7 +482,7 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: :func:`graphify.detect._to_relative_for_storage`. """ try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return # raw_calls (#: Pascal/Delphi cross-file inherited-call resolution) carries @@ -579,11 +596,11 @@ def _portability_anchors(path: "str | Path", root: "str | Path") -> tuple[list[s from graphify.ids import normalize_id try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return [], "", [], "" try: - path_resolved = Path(path).resolve() + path_resolved = _resolve_path(path) except (OSError, RuntimeError): path_resolved = Path(path) try: @@ -719,7 +736,7 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: absolute ``source_file`` values pass through unchanged. """ try: - root_resolved = Path(root).resolve() + root_resolved = _resolve_path(root) except OSError: return for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): @@ -758,14 +775,14 @@ def cache_dir(root: Path = Path("."), kind: str = "ast", vintage live. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out d = base / "cache" / kind if kind == "ast": d = d / f"v{_EXTRACTOR_VERSION}" _cleanup_stale_ast_entries(d.parent, d) elif prompt_fp: d = d / f"p{prompt_fp}" - d.mkdir(parents=True, exist_ok=True) + _make_dirs(d, exist_ok=True) return d @@ -813,13 +830,13 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", prompt_fp = _resolve_prompt_fp(prompt, prompt_file) entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" legacy_hit = False - if prompt_fp and not entry.exists() and allow_legacy: + if prompt_fp and not _path_exists(entry) and allow_legacy: legacy = cache_dir(location, kind) / f"{h}.json" - if legacy.exists(): + if _path_exists(legacy): entry, legacy_hit = legacy, True - if entry.exists(): + if _path_exists(entry): try: - result = json.loads(entry.read_text(encoding="utf-8")) + result = json.loads(_read_text(entry, encoding="utf-8")) except (json.JSONDecodeError, OSError): return None # A ``partial`` entry was produced from a truncated LLM response and @@ -874,7 +891,7 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a IsADirectoryError from aborting the whole batch. """ p = Path(path) - if not p.is_file(): + if not _path_is_file(p): return # Relativize source_file fields against ``root`` before write so the # cache file on disk is portable across machines and checkout @@ -905,17 +922,21 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a location = cache_root if cache_root is not None else root target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) entry = target_dir / f"{h}.json" - fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") + fd, tmp_path = tempfile.mkstemp( + dir=_io_path(target_dir), + prefix=f"{h}.", + suffix=".tmp", + ) try: os.write(fd, json.dumps(on_disk).encode()) os.close(fd) try: - os.replace(tmp_path, entry) + os.replace(tmp_path, _io_path(entry)) except PermissionError: # Windows: os.replace can fail with WinError 5 if the target is # briefly locked. Fall back to copy-then-delete. import shutil - shutil.copy2(tmp_path, entry) + shutil.copy2(tmp_path, _io_path(entry)) os.unlink(tmp_path) except Exception: try: @@ -931,38 +952,38 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a def cached_files(root: Path = Path(".")) -> set[str]: """Return set of file hashes that have a valid cache entry (any kind).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + base = _resolve_path(root) / _GRAPHIFY_OUT / "cache" hashes: set[str] = set() # Legacy flat entries - if base.is_dir(): - hashes.update(p.stem for p in base.glob("*.json")) + if _path_is_dir(base): + hashes.update(p.stem for p in _glob_paths(base, "*.json")) # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds # have per-prompt-fingerprint subdirs alongside pre-fingerprint flat entries # (#1939). for kind in ("ast", "semantic", "semantic-deep"): d = base / kind - if d.is_dir(): - hashes.update(p.stem for p in d.glob("**/*.json")) + if _path_is_dir(d): + hashes.update(p.stem for p in _glob_paths(d, "**/*.json")) return hashes def clear_cache(root: Path = Path(".")) -> None: """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy flat entries).""" - base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + base = _resolve_path(root) / _GRAPHIFY_OUT / "cache" # Legacy flat entries - if base.is_dir(): - for f in base.glob("*.json"): - f.unlink() + if _path_is_dir(base): + for f in _glob_paths(base, "*.json"): + _unlink_path(f) # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds # have per-prompt-fingerprint subdirs (#1939). for kind in ("ast", "semantic", "semantic-deep"): d = base / kind - if d.is_dir(): - for f in d.glob("**/*.json"): - f.unlink() + if _path_is_dir(d): + for f in _glob_paths(d, "**/*.json"): + _unlink_path(f) def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: @@ -1002,17 +1023,17 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: one doc on a future run, never incorrect output. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else _resolve_path(root) / _out pruned = 0 for kind in ("semantic", "semantic-deep"): semantic_dir = base / "cache" / kind - if not semantic_dir.is_dir(): + if not _path_is_dir(semantic_dir): continue - for entry in semantic_dir.glob("**/*.json"): + for entry in _glob_paths(semantic_dir, "**/*.json"): if entry.stem in live_hashes: continue try: - entry.unlink() + _unlink_path(entry) pruned += 1 except OSError: pass @@ -1172,7 +1193,7 @@ def save_semantic_cache( from collections import defaultdict kind = "semantic" if mode is None else f"semantic-{mode}" - root_path = Path(root).resolve() + root_path = _resolve_path(root) def _normalized(item: dict) -> dict: """Copy of ``item`` with a portable ``source_file`` (#2197). @@ -1214,7 +1235,7 @@ def resolved_source_path(value: str | Path) -> Path: if not path.is_absolute(): path = root_path / path try: - return path.resolve() + return _resolve_path(path) except (OSError, RuntimeError): # Keep the cache write best-effort for inaccessible paths or a # symlink loop emitted by an untrusted semantic result. @@ -1241,7 +1262,9 @@ def resolved_source_path(value: str | Path) -> Path: def group_skipped(fpath: str) -> bool: """Mirror the write-loop skip condition for one source_file group.""" p = resolved_source_path(fpath) - return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) + return not _path_is_file(p) or ( + allowed_paths is not None and p not in allowed_paths + ) # Dangling-reference pruning (#1916). A node group is skipped by the write # loop below when its source_file is not a real file (ghost path) or is @@ -1298,7 +1321,7 @@ def hyperedge_dangles(h: dict) -> bool: skipped_not_file = 0 for fpath, result in by_file.items(): p = resolved_source_path(fpath) - if p.is_file(): + if _path_is_file(p): if allowed_paths is not None and p not in allowed_paths: warnings.warn( "semantic cache skipped out-of-scope source_file " diff --git a/graphify/cargo_introspect.py b/graphify/cargo_introspect.py index fa03ed309..c5b24e45e 100644 --- a/graphify/cargo_introspect.py +++ b/graphify/cargo_introspect.py @@ -5,6 +5,13 @@ from pathlib import Path from typing import Any +from graphify.paths import ( + glob_paths as _glob_paths, + io_path as _io_path, + path_is_file as _path_is_file, + resolve_path as _resolve_path, +) + _CONFIDENCE_EXTRACTED = "EXTRACTED" @@ -20,7 +27,7 @@ def _load_toml(path: Path) -> dict[str, Any]: "--cargo on Python 3.10 needs tomli. Install with: pip install tomli" ) from None - with path.open("rb") as manifest: + with open(_io_path(path), "rb") as manifest: return tomllib.load(manifest) @@ -37,16 +44,16 @@ def _member_manifest_paths(root: Path, root_data: dict[str, Any]) -> list[Path]: for pattern in members: if not isinstance(pattern, str): continue - for member in sorted(root.glob(pattern)): + for member in sorted(_glob_paths(root, pattern)): manifest = member / "Cargo.toml" - if manifest.is_file() and manifest not in paths: + if _path_is_file(manifest) and manifest not in paths: paths.append(manifest) return paths def introspect_cargo(root: str | Path) -> dict[str, Any]: """Return crate nodes and internal dependency edges from Cargo manifests.""" - root_path = Path(root).resolve() + root_path = _resolve_path(root) root_manifest = root_path / "Cargo.toml" root_data = _load_toml(root_manifest) diff --git a/graphify/cli.py b/graphify/cli.py index 56c32140a..1eb476d47 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -11,7 +11,17 @@ import re import sys import time -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_text as _read_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_text, +) from pathlib import Path @@ -123,7 +133,7 @@ def _resolve(value: str) -> Path: if not p.is_absolute(): p = root / p try: - return p.resolve() + return _resolve_path(p) except (OSError, RuntimeError): return p @@ -191,19 +201,19 @@ def _stale_graph_sources( """ from graphify.paths import nfc try: - data = json.loads(graph_path.read_text(encoding="utf-8")) + data = json.loads(_read_text(graph_path, encoding="utf-8")) except Exception: return [] if not isinstance(data, dict): return [] try: - root_res = scan_root.resolve() + root_res = _resolve_path(scan_root) except (OSError, RuntimeError): root_res = scan_root # /graphify-out/graph.json — relative source_files may be anchored here. out_base = graph_path.parent.parent try: - out_base = out_base.resolve() + out_base = _resolve_path(out_base) except (OSError, RuntimeError): pass @@ -214,7 +224,7 @@ def _within_root(p: Path) -> bool: except ValueError: pass try: - p.resolve().relative_to(root_res) + _resolve_path(p).relative_to(root_res) return True except (ValueError, OSError, RuntimeError): return False @@ -226,7 +236,7 @@ def _in_seen(p: Path) -> bool: if nfc(str(p)) in seen_nfc: return True try: - return nfc(str(p.resolve())) in seen_nfc + return nfc(str(_resolve_path(p))) in seen_nfc except (OSError, RuntimeError): return False @@ -252,7 +262,7 @@ def _in_seen(p: Path) -> bool: def _provably_excluded(c: Path) -> bool: spellings = [nfc(str(c))] try: - spellings.append(nfc(str(c.resolve()))) + spellings.append(nfc(str(_resolve_path(c)))) except (OSError, RuntimeError): pass for s in spellings: @@ -301,7 +311,7 @@ def _provably_excluded(c: Path) -> bool: alive = [] for c in in_root: try: - if c.exists(): + if _path_exists(c): alive.append(c) except OSError: pass @@ -563,7 +573,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: # candidate that resolves outside that root is out-of-project. root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) try: - root = root.resolve() + root = _resolve_path(root) except (OSError, RuntimeError): pass path_vals = [str(t.get("file_path") or ""), str(t.get("path") or "")] @@ -576,7 +586,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: in_project = True # relative -> anchored at cwd == in project break try: - p.resolve().relative_to(root) + _resolve_path(p).relative_to(root) in_project = True break except (ValueError, OSError, RuntimeError): @@ -585,7 +595,7 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: return # One stat for existence + mtime of the graph. try: - gmtime = os.stat(str(out_path("graph.json"))).st_mtime + gmtime = _path_stat(out_path("graph.json")).st_mtime except OSError: return # #1840 (b): stale-for-target -> soften, never block. The target file @@ -594,11 +604,11 @@ def _run_hook_guard(kind: str, strict: bool = False) -> None: fp = str(t.get("file_path") or "") if fp: try: - stale = os.stat(fp).st_mtime > gmtime + stale = _path_stat(fp).st_mtime > gmtime except OSError: stale = False try: - if out_path("needs_update").exists(): + if _path_exists(out_path("needs_update")): stale = True except Exception: pass @@ -629,16 +639,16 @@ def _target_is_indexed(file_path: str, root: "Path") -> bool: return True try: mp = out_path("manifest.json") - st = mp.stat() + st = _path_stat(mp) if st.st_size > 2_000_000: return True - manifest = json.loads(mp.read_text(encoding="utf-8")) + manifest = json.loads(_read_text(mp, encoding="utf-8")) if not isinstance(manifest, dict) or not manifest: return True p = Path(file_path) rels = set() try: - rels.add(p.resolve().relative_to(root).as_posix()) + rels.add(_resolve_path(p).relative_to(root).as_posix()) except (ValueError, OSError, RuntimeError): pass rels.add(p.name) @@ -1514,7 +1524,7 @@ def dispatch_command(cmd: str) -> None: summary = diagnose_file( graph_path, directed=directed, - root=Path(".").resolve(), + root=_resolve_path(Path(".")), max_examples=max_examples, extract_path=extract_path, ) @@ -1564,7 +1574,7 @@ def dispatch_command(cmd: str) -> None: elif cmd == "watch": watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): + if not _path_exists(watch_path): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import watch as _watch @@ -1638,7 +1648,7 @@ def dispatch_command(cmd: str) -> None: if watch_path is None: watch_path = Path(".") graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): + if not _path_exists(graph_json): print( f"error: no graph found at {graph_json} — run /graphify first", file=sys.stderr, @@ -1668,7 +1678,7 @@ def dispatch_command(cmd: str) -> None: except ValueError: _over_cap = True try: - _over_cap_bytes = graph_json.stat().st_size + _over_cap_bytes = _path_stat(graph_json).st_size except OSError: _over_cap_bytes = -1 print( @@ -1676,7 +1686,7 @@ def dispatch_command(cmd: str) -> None: f"falling back to community-aggregation view (node_limit=5000)", file=sys.stderr, ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) + _raw = json.loads(_read_text(graph_json, encoding="utf-8")) _directed = bool(_raw.get("directed", False)) G = build_from_json(_raw, directed=_directed) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") @@ -1924,11 +1934,11 @@ def dispatch_command(cmd: str) -> None: else: # Try to recover the scan root saved by the last full build saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) + if _path_exists(saved): + watch_path = Path(_read_text(saved, encoding="utf-8").strip()) else: watch_path = Path(".") - if not watch_path.exists(): + if not _path_exists(watch_path): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import _rebuild_code @@ -1978,7 +1988,7 @@ def dispatch_command(cmd: str) -> None: sys.exit(1) from graphify.watch import check_update - check_update(Path(sys.argv[2]).resolve()) + check_update(_resolve_path(sys.argv[2])) sys.exit(0) elif cmd == "tree": # Emit a D3 v7 collapsible-tree HTML view of graph.json: @@ -2615,10 +2625,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": has_path = True if sys.argv[2].startswith("-"): has_path = False - target = Path(".").resolve() + target = _resolve_path(Path(".")) else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): + target = _resolve_path(Path(sys.argv[2])) + if not _path_exists(target): print(f"error: path not found: {target}", file=sys.stderr) sys.exit(1) @@ -2779,9 +2789,9 @@ def _parse_float(name: str, raw: str) -> float: # Resolve output dir. The user-facing contract is "/graphify-out/" # so a fresh checkout writes graphify-out/ at the project root, matching # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) + out_root = (_resolve_path(out_dir) if out_dir else target) graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) + _make_dirs(graphify_out, exist_ok=True) # Persist corpus-shaping options so later update/watch/hook rebuilds # use the same file set as the initial extraction (#1886). from graphify.watch import ( @@ -2822,13 +2832,13 @@ def _parse_float(name: str, raw: str) -> float: # and genuinely-deleted sources against the current corpus, so doc/ # paper/image nodes survive a --code-only rebuild instead of being # dropped with the rest of the committed graph. - incremental_mode = existing_graph_path.exists() if has_path else False + incremental_mode = _path_exists(existing_graph_path) if has_path else False # --force: full scan, not the manifest-gated incremental diff — a warm # unchanged tree would otherwise dispatch zero files (#1894). incremental_mode = incremental_mode and not force if force: print("[graphify extract] --force: full re-scan, semantic cache reads skipped") - elif incremental_mode and not manifest_path.exists(): + elif incremental_mode and not _path_exists(manifest_path): print( "[graphify extract] manifest.json missing; using existing " "graph.json as the incremental baseline (all files re-checked; " @@ -3263,7 +3273,7 @@ def _progress(idx: int, total: int, _result: dict) -> None: _abs = Path(_fp) if not _abs.is_absolute(): _abs = Path(target) / _abs - if not _abs.is_file(): + if not _path_is_file(_abs): continue # deleted/missing — leave out so its entry is pruned try: _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) @@ -3355,7 +3365,7 @@ def _invalidate_file_manifest_for_db_graph() -> None: if has_path: return try: - manifest_path.unlink(missing_ok=True) + _unlink_path(manifest_path, missing_ok=True) except OSError as exc: print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) sys.exit(1) @@ -3484,8 +3494,10 @@ def _invalidate_file_manifest_for_db_graph() -> None: # relativize deleted-file paths correctly even for a custom --out # (its grandparent-of-graph.json fallback points at the wrong dir # otherwise, and deleted files never prune — #2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" + _write_text( + graphify_out / ".graphify_root", + str(_resolve_path(target)), + encoding="utf-8", ) except OSError: pass @@ -3618,15 +3630,19 @@ def _invalidate_file_manifest_for_db_graph() -> None: try: # See the --no-cluster path above: persist the scan root so build_merge # can relativize deleted-file paths under a custom --out (#2012/#1571). - (graphify_out / ".graphify_root").write_text( - str(Path(target).resolve()), encoding="utf-8" + _write_text( + graphify_out / ".graphify_root", + str(_resolve_path(target)), + encoding="utf-8", ) except OSError: pass stages.mark("export") if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + _write_text( + graphify_out / ".graphify_semantic_marker", + json.dumps({"output_tokens": merged["output_tokens"]}), + encoding="utf-8", ) if global_merge: from graphify.global_graph import global_add as _global_add @@ -3737,19 +3753,20 @@ def _invalidate_file_manifest_for_db_graph() -> None: i += 1 else: i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + files = [f for f in _read_text(files_from, encoding="utf-8").splitlines() if f.strip()] cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( files, root, mode=cache_mode, prompt_file=prompt_file ) out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) + _make_dirs(out, exist_ok=True) if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( + _write_text( + out / ".graphify_cached.json", json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, ensure_ascii=False), encoding="utf-8", ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + _write_text(out / ".graphify_uncached.txt", "\n".join(uncached), encoding="utf-8") print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") elif cmd == "merge-chunks": @@ -3857,8 +3874,16 @@ def _invalidate_file_manifest_for_db_graph() -> None: print("error: --out required", file=sys.stderr) sys.exit(1) empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + cached_data = ( + json.loads(_read_text(cached_path, encoding="utf-8")) + if cached_path and _path_exists(cached_path) + else empty + ) + new_data = ( + json.loads(_read_text(new_path, encoding="utf-8")) + if new_path and _path_exists(new_path) + else empty + ) seen_ids2: set[str] = set() all_nodes: list[dict] = [] for n in cached_data.get("nodes", []) + new_data.get("nodes", []): @@ -3870,12 +3895,16 @@ def _invalidate_file_manifest_for_db_graph() -> None: "edges": cached_data.get("edges", []) + new_data.get("edges", []), "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), } - out_path2.parent.mkdir(parents=True, exist_ok=True) + _make_dirs(out_path2.parent, exist_ok=True) from graphify.paths import write_json_atomic as _wja _wja(out_path2, merged2, ensure_ascii=False) print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): + elif ( + _path_exists(Path(cmd).expanduser()) + or cmd in (".", "..") + or cmd.startswith(("./", "../", "/", "~")) + ): # User ran `graphify ` directly — treat as `graphify extract `. # Common when following the PowerShell note in README (`graphify .`) or # copy-pasting skill invocations without the leading slash. diff --git a/graphify/detect.py b/graphify/detect.py index f8e185436..e17788188 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -15,7 +15,21 @@ convert_google_workspace_file, google_workspace_enabled, ) -from graphify.paths import GRAPHIFY_OUT, out_path +from graphify.paths import ( + GRAPHIFY_OUT, + io_path as _os_path, + make_dirs as _make_dirs, + out_path, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_is_symlink as _path_is_symlink, + path_stat as _path_stat, + read_text as _read_text, + resolve_path as _resolve_path, + walk_path as _walk_path, + write_text as _write_text, +) class FileType(str, Enum): @@ -52,7 +66,7 @@ class FileType(str, Enum): def _file_within_size_cap(path: Path, cap: int = _OFFICE_MAX_RAW_BYTES) -> bool: """True if *path* exists and its on-disk size is within *cap*.""" try: - return path.stat().st_size <= cap + return _path_stat(path).st_size <= cap except OSError: return False @@ -72,7 +86,7 @@ def _zip_within_caps(path: Path) -> bool: if not _file_within_size_cap(path): return False try: - with zipfile.ZipFile(path) as zf: + with zipfile.ZipFile(_os_path(path)) as zf: infos = zf.infolist() compressed = sum(i.compress_size for i in infos) or 1 declared = sum(i.file_size for i in infos) @@ -296,7 +310,7 @@ def _looks_like_paper(path: Path) -> bool: """Heuristic: does this text file read like an academic paper?""" try: # Only scan first 3000 chars for speed - text = path.read_text(encoding="utf-8", errors="ignore")[:3000] + text = _read_text(path, encoding="utf-8", errors="ignore")[:3000] hits = sum(1 for pattern in _PAPER_SIGNALS if pattern.search(text)) return hits >= _PAPER_SIGNAL_THRESHOLD except Exception: @@ -460,7 +474,7 @@ def _shebang_interpreter(path: Path) -> str | None: no shebang / the file is unreadable / parsing fails. """ try: - with path.open("rb") as f: + with open(_os_path(path), "rb") as f: first = f.read(256) if not first.startswith(b"#!"): return None @@ -530,7 +544,7 @@ def extract_pdf_text(path: Path) -> str: return "" try: from pypdf import PdfReader - reader = PdfReader(str(path)) + reader = PdfReader(_os_path(path)) pages = [] for page in reader.pages: text = page.extract_text() @@ -548,7 +562,7 @@ def docx_to_markdown(path: Path) -> str: try: from docx import Document from docx.oxml.ns import qn - doc = Document(str(path)) + doc = Document(_os_path(path)) lines = [] for para in doc.paragraphs: style = para.style.name if para.style else "" @@ -589,7 +603,7 @@ def xlsx_to_markdown(path: Path) -> str: return "" try: import openpyxl - wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True) + wb = openpyxl.load_workbook(_os_path(path), read_only=True, data_only=True) sections = [] for sheet_name in wb.sheetnames: ws = wb[sheet_name] @@ -630,7 +644,7 @@ def _nid(*parts: str) -> str: return {"nodes": [], "edges": []} try: - wb = openpyxl.load_workbook(str(path), read_only=False, data_only=True) + wb = openpyxl.load_workbook(_os_path(path), read_only=False, data_only=True) except Exception: return {"nodes": [], "edges": []} @@ -722,7 +736,7 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - if not text.strip(): return None - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) # Use a stable name derived from the original path to avoid collisions. # Hash the path RELATIVE to the scan root, not the absolute path: the # absolute form salts the name with the checkout location, so the same @@ -741,12 +755,12 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - # Default layout: out_dir is //converted. root = out_dir.parent.parent try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() + key = _resolve_path(path).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): # Not under the scan root (custom GRAPHIFY_OUT layouts, --include # sources, direct API callers): keep the previous absolute form rather # than guessing, so behavior is unchanged for those cases. - key = str(path.resolve()) + key = str(_resolve_path(path)) normalized_path = unicodedata.normalize("NFC", key) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" @@ -758,12 +772,16 @@ def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) - # incremental hash check then correctly picks up. An unchanged source keeps # its (newer-or-equal) sidecar untouched so it never churns (#1226). try: - if out_path.exists() and os.stat(_os_path(out_path)).st_mtime >= os.stat(_os_path(path)).st_mtime: + if ( + _path_exists(out_path) + and _path_stat(out_path).st_mtime >= _path_stat(path).st_mtime + ): return out_path except OSError: - if out_path.exists(): + if _path_exists(out_path): return out_path - out_path.write_text( + _write_text( + out_path, f"\n\n{text}", encoding="utf-8", ) @@ -850,10 +868,10 @@ def _has_coverage_artifacts(d: "Path") -> bool: """ try: for name in _COVERAGE_ARTIFACT_FILES: - if (d / name).is_file(): + if _path_is_file(d / name): return True for name in _COVERAGE_ARTIFACT_DIRS: - if (d / name).is_dir(): + if _path_is_dir(d / name): return True except OSError: pass @@ -870,13 +888,19 @@ def _has_venv_markers(d: "Path") -> bool: ``conda-meta/`` (``conda create -p ./env`` writes no pyvenv.cfg). """ try: - if (d / "pyvenv.cfg").is_file(): - return True - if (d / "bin" / "activate").is_file() or (d / "Scripts" / "activate").is_file(): + if _path_is_file(d / "pyvenv.cfg"): return True - if next(d.glob("lib/python*"), None) is not None: + if _path_is_file(d / "bin" / "activate") or _path_is_file( + d / "Scripts" / "activate" + ): return True - if (d / "conda-meta").is_dir(): + try: + with os.scandir(_os_path(d / "lib")) as entries: + if any(entry.name.startswith("python") for entry in entries): + return True + except OSError: + pass + if _path_is_dir(d / "conda-meta"): return True except OSError: pass @@ -907,8 +931,9 @@ def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool: if parent.name in _JS_SNAPSHOT_TEST_ROOTS: return True try: - if next(snap_dir.glob("*.snap"), None) is not None: - return True + with os.scandir(_os_path(snap_dir)) as entries: + if any(entry.name.endswith(".snap") for entry in entries): + return True except OSError: pass return False @@ -953,10 +978,10 @@ def _parse_gitignore_line(raw: str) -> str: def _find_vcs_root(start: Path) -> Path | None: """Walk upward from start; return the first directory containing a VCS marker.""" - current = start.resolve() + current = _resolve_path(start) home = Path.home() while True: - if any((current / m).exists() for m in _VCS_MARKERS): + if any(_path_exists(current / m) for m in _VCS_MARKERS): return current parent = current.parent if parent == current or current == home: @@ -977,33 +1002,33 @@ def _git_info_exclude(vcs_root: Path) -> Path | None: """ dot_git = vcs_root / ".git" git_dir: Path | None = None - if dot_git.is_dir(): + if _path_is_dir(dot_git): git_dir = dot_git - elif dot_git.is_file(): + elif _path_is_file(dot_git): try: - content = dot_git.read_text(encoding="utf-8", errors="ignore").strip() + content = _read_text(dot_git, encoding="utf-8", errors="ignore").strip() except OSError: content = "" if content.startswith("gitdir:"): gd = Path(content[len("gitdir:"):].strip()) if not gd.is_absolute(): - gd = (vcs_root / gd).resolve() + gd = _resolve_path(vcs_root / gd) git_dir = gd # A linked worktree's gitdir holds a `commondir` file pointing at the # shared git dir, where info/exclude actually lives. commondir = gd / "commondir" - if commondir.exists(): + if _path_exists(commondir): try: - cd_raw = commondir.read_text(encoding="utf-8", errors="ignore").strip() + cd_raw = _read_text(commondir, encoding="utf-8", errors="ignore").strip() except OSError: cd_raw = "" if cd_raw: cd = Path(cd_raw) - git_dir = cd if cd.is_absolute() else (gd / cd).resolve() + git_dir = cd if cd.is_absolute() else _resolve_path(gd / cd) if git_dir is None: return None exclude = git_dir / "info" / "exclude" - return exclude if exclude.is_file() else None + return exclude if _path_is_file(exclude) else None def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: @@ -1025,8 +1050,8 @@ def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, patterns: list[tuple[Path, str]] = [] for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)): ignore_file = d / fname - if ignore_file.exists(): - for raw in ignore_file.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + if _path_exists(ignore_file): + for raw in _read_text(ignore_file, encoding="utf-8-sig", errors="ignore").splitlines(): line = _parse_gitignore_line(raw) if line: patterns.append((d, line)) @@ -1047,7 +1072,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa scan root are picked up live during the os.walk in `detect()` instead, since they aren't known until the walk reaches them (#1206). """ - root = root.resolve() + root = _resolve_path(root) ceiling = _find_vcs_root(root) or root # Collect ancestor dirs from ceiling down to root (outer → inner) @@ -1068,7 +1093,7 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa # re-include still override it (#1810). info_exclude = _git_info_exclude(ceiling) if gitignore else None if info_exclude is not None: - for raw in info_exclude.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + for raw in _read_text(info_exclude, encoding="utf-8-sig", errors="ignore").splitlines(): line = _parse_gitignore_line(raw) if line: patterns.append((ceiling, line)) @@ -1170,7 +1195,7 @@ def _matches(rel: str, p: str, path_relative: bool) -> bool: continue # target outside this pattern's anchor: cannot match if rel_anchor != ".": matched = _matches(rel_anchor, p, path_relative=path_relative) - if matched and directory_only and not target.is_dir(): + if matched and directory_only and not _path_is_dir(target): matched = False if matched: @@ -1204,9 +1229,10 @@ def _auto_follow_symlinks(root: Path) -> bool: explicit opt-in, and out-of-root symlink targets are never indexed. """ try: - for p in root.iterdir(): - if p.is_symlink(): - return True + with os.scandir(_os_path(root)) as entries: + for entry in entries: + if entry.is_symlink(): + return True except (OSError, PermissionError): pass return False @@ -1215,18 +1241,18 @@ def _auto_follow_symlinks(root: Path) -> bool: def _resolves_under_root(path: Path, root: Path) -> bool: """True when ``path`` resolves to a target inside ``root``.""" try: - path.resolve().relative_to(root.resolve()) + _resolve_path(path).relative_to(_resolve_path(root)) except (OSError, RuntimeError, ValueError): return False return True def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: - root = root.resolve() + root = _resolve_path(root) configured_out_dir = root / GRAPHIFY_OUT configured_out_names = {configured_out_dir.name} try: - configured_out_dir = configured_out_dir.resolve() + configured_out_dir = _resolve_path(configured_out_dir) except (OSError, RuntimeError): configured_out_dir = configured_out_dir.absolute() configured_out_names.add(configured_out_dir.name) @@ -1234,7 +1260,7 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: # no consumers, so the file has been a silent no-op since dot directories # became indexed by default (#873). Surface that once per scan so a # leftover allowlist file is not a silent behavior change. - if (root / ".graphifyinclude").is_file(): + if _path_is_file(root / ".graphifyinclude"): import sys as _sys print( "[graphify] WARNING: .graphifyinclude is no longer supported " @@ -1284,7 +1310,7 @@ def _wc(path: Path) -> int: # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / GRAPHIFY_OUT / "memory" scan_paths = [root] - if memory_dir.exists(): + if _path_exists(memory_dir): scan_paths.append(memory_dir) seen: set[Path] = set() @@ -1309,14 +1335,20 @@ def _on_walk_error(err: OSError) -> None: ) for scan_root in scan_paths: - in_memory_tree = memory_dir.exists() and str(scan_root).startswith(str(memory_dir)) - for dirpath, dirnames, filenames in os.walk( + in_memory_tree = _path_exists(memory_dir) and str(scan_root).startswith( + str(memory_dir) + ) + for dirpath, dirnames, filenames in _walk_path( scan_root, followlinks=follow_symlinks, onerror=_on_walk_error ): dp = Path(dirpath) - if follow_symlinks and os.path.islink(dirpath): - real = os.path.realpath(dirpath) - parent_real = os.path.realpath(os.path.dirname(dirpath)) + # os.walk must stay in the extended Windows namespace so every + # recursive scandir call can reach long descendants. Convert the + # yielded path back immediately: graph/cache identities must use the + # ordinary UNC/drive spelling, not the transport-only \\?\ prefix. + if follow_symlinks and _path_is_symlink(dp): + real = str(_resolve_path(dp)) + parent_real = str(_resolve_path(dp.parent)) if parent_real == real or parent_real.startswith(real + os.sep): dirnames.clear() continue @@ -1347,7 +1379,7 @@ def _on_walk_error(err: OSError) -> None: is_configured_out = False if d in configured_out_names: try: - is_configured_out = child.resolve() == configured_out_dir + is_configured_out = _resolve_path(child) == configured_out_dir except (OSError, RuntimeError): pass if is_configured_out: @@ -1368,7 +1400,7 @@ def _on_walk_error(err: OSError) -> None: safe_dirs: list[str] = [] for d in dirnames: child = dp / d - if child.is_symlink() and not _resolves_under_root(child, root): + if _path_is_symlink(child) and not _resolves_under_root(child, root): skipped_sensitive.append(str(child) + " [symlink target outside scan root]") continue safe_dirs.append(d) @@ -1387,7 +1419,7 @@ def _on_walk_error(err: OSError) -> None: for p in all_files: # For memory dir files, skip hidden/noise filtering - in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir)) + in_memory = _path_exists(memory_dir) and str(p).startswith(str(memory_dir)) if not in_memory: # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): @@ -1479,38 +1511,10 @@ def _on_walk_error(err: OSError) -> None: "ignored": sorted(ignored), "pruned_noise_dirs": sorted(pruned_noise), "graphifyignore_patterns": len(ignore_patterns), - "scan_root": str(root.resolve()), + "scan_root": str(_resolve_path(root)), } -def _os_path(path: Path) -> str: - r"""Return an OS path string safe for open()/stat() on Windows long paths. - - On win32, paths longer than the legacy MAX_PATH (260 chars) are rejected by - the plain file APIs unless prefixed with the extended-length marker ``\\?\`` - (which also requires a fully-qualified path). Without it, _md5_file / - save_manifest / count_words silently fail to hash deeply-nested files, so - their manifest entry never stabilizes and detect_incremental re-flags them - as changed on every run (#1655). cache._normalize_path strips this prefix - for stable KEYS; this adds it for I/O. Non-win32 and already-prefixed paths - pass through unchanged. - """ - import sys - if sys.platform != "win32": - return str(path) - s = str(path) - if s.startswith("\\\\?\\"): - return s - try: - s = os.path.abspath(s) # \\?\ requires a fully-qualified path - except Exception: - return str(path) - if s.startswith("\\\\"): - # UNC share \\server\share -> \\?\UNC\server\share - return "\\\\?\\UNC\\" + s[2:] - return "\\\\?\\" + s - - def _md5_file(path: Path) -> str: """MD5 of file contents streamed in 64KB chunks — for change detection only.""" import hashlib as _hl @@ -1528,7 +1532,7 @@ def _stat_and_hash(path_str: str) -> tuple[str, float, str] | None: """Stat + MD5 a single file; returns None on OSError (e.g. deleted mid-run).""" try: p = Path(path_str) - return path_str, os.stat(_os_path(p)).st_mtime, _md5_file(p) + return path_str, _path_stat(p).st_mtime, _md5_file(p) except OSError: return None @@ -1569,7 +1573,7 @@ def _to_relative_for_storage(key: str, root: Path) -> str: if not p.is_absolute(): return key try: - base = _nfc(str(Path(root).resolve())) + base = _nfc(str(_resolve_path(root))) rel = os.path.relpath(_nfc(str(p)), base) except (ValueError, OSError): return key # outside root (e.g. Windows cross-drive) @@ -1597,7 +1601,7 @@ def _to_absolute_from_storage(key: str, root: Path) -> str: return str(p) # NFC the joined result so an NFD-resolved root + relative key lands on # the same form load_manifest / detect_incremental compare against. - return _nfc(str(Path(root).resolve() / p)) + return _nfc(str(_resolve_path(root) / p)) def load_manifest( @@ -1617,7 +1621,7 @@ def load_manifest( form still matches a scan that yields the other (#2221). """ try: - raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + raw = json.loads(_read_text(manifest_path, encoding="utf-8")) except Exception: return {} if not isinstance(raw, dict): @@ -1686,7 +1690,7 @@ def _path_index(paths: set[str] | list[str] | None) -> set[str] | None: scan_set = _path_index(scan_corpus) clear_set = _path_index(clear_semantic) try: - root_res: Path | None = Path(root).resolve() if root is not None else None + root_res: Path | None = _resolve_path(root) if root is not None else None except (OSError, RuntimeError): root_res = Path(root) if root is not None else None @@ -1694,7 +1698,7 @@ def _in_scan(path_str: str) -> bool: if path_str in scan_set or _nfc(path_str) in scan_set: return True try: - resolved = str(Path(path_str).resolve()) + resolved = str(_resolve_path(path_str)) return resolved in scan_set or _nfc(resolved) in scan_set except (OSError, RuntimeError): return False @@ -1703,7 +1707,7 @@ def _in_clear(path_str: str) -> bool: if path_str in clear_set or _nfc(path_str) in clear_set: return True try: - resolved = str(Path(path_str).resolve()) + resolved = str(_resolve_path(path_str)) return resolved in clear_set or _nfc(resolved) in clear_set except (OSError, RuntimeError): return False @@ -1720,7 +1724,7 @@ def _in_root(path_str: str) -> bool: except ValueError: pass try: - p.resolve().relative_to(root_res) + _resolve_path(p).relative_to(root_res) return True except (ValueError, OSError, RuntimeError): return False @@ -1747,7 +1751,7 @@ def _normalise_entry(entry): if normalised is None: continue try: - if not Path(f).exists(): + if not _path_exists(f): continue except OSError: continue @@ -1861,7 +1865,7 @@ def detect_incremental( # Manifest keys are NFC; scan paths may arrive NFD (#2221). stored = manifest.get(_nfc(f)) try: - current_mtime = os.stat(_os_path(Path(f))).st_mtime + current_mtime = _path_stat(f).st_mtime except Exception: current_mtime = 0 @@ -1917,7 +1921,7 @@ def detect_incremental( if _nfc(f) in current_files: continue try: - alive = Path(f).exists() + alive = _path_exists(f) except OSError: alive = False (excluded_files if alive else deleted_files).append(f) diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index fcb9a11cf..6ce9405bb 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -11,6 +11,11 @@ import networkx as nx +from graphify.paths import ( + path_exists as _path_exists, + read_text as _read_file_text, +) + _SUPPRESSION_DECL_RE = re.compile(r"^\s*(?Pseen_[A-Za-z0-9_]+)\s*[:=]") _TYPE_TUPLE_RE = re.compile(r"set\[tuple\[(?P[^\]]+)\]\]") @@ -122,7 +127,7 @@ def _tuple_arity_from_annotation(line: str) -> int: def scan_producer_suppression_sites(path: str | Path) -> dict[str, Any]: """Find likely `seen_*` producer-suppression sets in an extractor file.""" source_path = Path(path) - if not source_path.exists(): + if not _path_exists(source_path): return { "path": str(source_path), "total_sites": 0, @@ -131,7 +136,7 @@ def scan_producer_suppression_sites(path: str | Path) -> dict[str, Any]: } sites: list[dict[str, Any]] = [] - lines = source_path.read_text(encoding="utf-8").splitlines() + lines = _read_file_text(source_path, encoding="utf-8").splitlines() for lineno, line in enumerate(lines, start=1): match = _SUPPRESSION_DECL_RE.match(line) if not match: @@ -284,7 +289,7 @@ def _read_json_file(path: str | Path) -> dict[str, Any]: json_path = Path(path) check_graph_file_size_cap(json_path) try: - data = json.loads(json_path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(json_path, encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: raise RuntimeError( f"Cannot parse {json_path}: {exc}. " diff --git a/graphify/extract.py b/graphify/extract.py index 30f31d329..08a90d288 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -56,7 +56,17 @@ from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata -from graphify.paths import disambiguate_ambiguous_candidates +from graphify.paths import ( + disambiguate_ambiguous_candidates, + iterdir_path as _iterdir_path, + path_exists as _path_exists, + path_is_file as _path_is_file, + path_is_symlink as _path_is_symlink, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, + walk_path as _walk_path, +) from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 @@ -200,7 +210,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: (ambiguous -> leave dangling, as before). Files whose package root IS the scan root are skipped (ids already coincide).""" try: - root = Path(root).resolve() + root = _resolve_path(root) except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} @@ -209,17 +219,17 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: if p.suffix.lower() not in (".py", ".pyi"): continue try: - rel = Path(p).resolve().relative_to(root) + rel = _resolve_path(p).relative_to(root) except (ValueError, OSError): continue parts = rel.parts if len(parts) < 2: continue # top-level file: scan-root-relative id already matches - d = Path(p).resolve().parent + d = _resolve_path(p).parent levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. - while levels < len(parts) - 1 and (d / "__init__.py").is_file(): + while levels < len(parts) - 1 and _path_is_file(d / "__init__.py"): levels += 1 d = d.parent if levels == 0: @@ -373,7 +383,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s # and popped before graph.json ships. if target_path is not None: try: - if target_path.is_file(): + if _path_is_file(target_path): edge["target_file"] = str(target_path) except OSError: pass @@ -1086,7 +1096,7 @@ def _extract_python_rationale(path: Path, result: dict) -> None: from tree_sitter import Language, Parser language = Language(tspython.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception: @@ -1242,7 +1252,7 @@ def _extract_js_rationale(path: Path, result: dict) -> None: Mutates result in-place by appending to result['nodes'] and result['edges']. """ try: - source_text = path.read_text(encoding="utf-8", errors="replace") + source_text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception: return @@ -1352,7 +1362,7 @@ def _emit_rescued_import( ) node_id = _make_id(str(resolved)) stub_source_file = str(resolved) - if resolved is not None and resolved.is_file(): + if resolved is not None and _path_is_file(resolved): resolved_file = resolved else: # Check tsconfig.json path aliases (e.g. "$lib/" -> "src/lib/", @@ -1364,7 +1374,7 @@ def _emit_rescued_import( resolved_alias = _resolve_js_module_path(resolved_alias) node_id = _make_id(str(resolved_alias)) stub_source_file = str(resolved_alias) - if resolved_alias is not None and resolved_alias.is_file(): + if resolved_alias is not None and _path_is_file(resolved_alias): resolved_file = resolved_alias else: # Bare/scoped import (node_modules) - use last segment; @@ -1408,7 +1418,7 @@ def extract_svelte(path: Path) -> dict: result = _extract_generic(path, _JS_CONFIG) try: import re as _re - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") existing_ids = {n["id"] for n in result.get("nodes", [])} # Source file node ID must match the one _extract_generic creates: # _make_id(str(path)) - single arg, no stem prefix. Otherwise the source @@ -1470,7 +1480,7 @@ def extract_astro(path: Path) -> dict: result = _extract_generic(path, _JS_CONFIG) try: import re as _re - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") existing_ids = {n["id"] for n in result.get("nodes", [])} file_node_id = _make_id(str(path)) aliases = _load_tsconfig_aliases(path.parent) @@ -1531,7 +1541,7 @@ def extract_vue(path: Path) -> dict: ``import('…')`` dynamic imports the AST does not edge. """ try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": []} @@ -1576,7 +1586,7 @@ def _is_spock_file(path: Path, ts_result: dict) -> bool: import re as _re _SPOCK_FEATURE_RE = _re.compile(r"""^\s*def\s+[\"']""", _re.MULTILINE) try: - return bool(_SPOCK_FEATURE_RE.search(path.read_text(errors="replace"))) + return bool(_SPOCK_FEATURE_RE.search(_read_file_text(path, errors="replace"))) except OSError: return False @@ -1587,7 +1597,7 @@ def _extract_spock_fallback(path: Path, ts_result: dict) -> dict: (which survive reliably) with class and feature-method nodes extracted via regex. """ import re as _re - source = path.read_text(errors="replace") + source = _read_file_text(path, errors="replace") str_path = str(path) stem = _file_stem(path) @@ -3169,7 +3179,7 @@ def extract_lazarus_package(path: Path) -> dict: """ try: import xml.etree.ElementTree as ET - src = path.read_bytes() + src = _read_file_bytes(path) except OSError as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -3276,7 +3286,7 @@ def extract_slnx(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -3306,7 +3316,7 @@ def extract_slnx(path: Path) -> dict: def _resolve(proj_path: str) -> str: proj_path = proj_path.replace("\\", "/") try: - return str((path.parent / proj_path).resolve()) + return str(_resolve_path(path.parent / proj_path)) except Exception: return proj_path @@ -3355,7 +3365,7 @@ def extract_csproj(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -3435,7 +3445,7 @@ def find_all(tag: str): continue ref_path_norm = ref_path.replace("\\", "/") try: - abs_ref = str((path.parent / ref_path_norm).resolve()) + abs_ref = str(_resolve_path(path.parent / ref_path_norm)) except Exception: abs_ref = ref_path_norm proj_nid = _make_id(abs_ref) @@ -3570,10 +3580,10 @@ def _xaml_binding_refs(value: str) -> tuple[str | None, str | None]: def _xaml_codebehind_path(path: Path) -> Path | None: expected = path.with_suffix(path.suffix + ".cs") - if expected.exists(): + if _path_exists(expected): return expected try: - for sibling in path.parent.iterdir(): + for sibling in _iterdir_path(path.parent): if sibling.name.casefold() == expected.name.casefold(): return sibling except OSError: @@ -3616,7 +3626,7 @@ def _xaml_codebehind_symbols( # parameter list on method nodes, so we read it from the code-behind source # at the method's recorded line. try: - cb_lines = codebehind.read_text(encoding="utf-8", errors="replace").splitlines() + cb_lines = _read_file_text(codebehind, encoding="utf-8", errors="replace").splitlines() except OSError: cb_lines = [] @@ -3714,16 +3724,16 @@ def _xaml_project_root(path: Path) -> Path: root = path.parent for directory in (path.parent, *path.parent.parents): try: - if any(child.suffix in project_markers for child in directory.iterdir()): + if any(child.suffix in project_markers for child in _iterdir_path(directory)): root = directory break except OSError: continue if _XAML_ACTIVE_EXTRACT_ROOT is None: return root - boundary = _XAML_ACTIVE_EXTRACT_ROOT.resolve() + boundary = _resolve_path(_XAML_ACTIVE_EXTRACT_ROOT) try: - root.resolve().relative_to(boundary) + _resolve_path(root).relative_to(boundary) return root except ValueError: return boundary @@ -3732,7 +3742,7 @@ def _xaml_project_root(path: Path) -> Path: def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore root = _xaml_project_root(path) - cache_key = str(root.resolve()) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None + cache_key = str(_resolve_path(root)) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None if cache_key and cache_key in _XAML_CSHARP_CLASS_CACHE: return _XAML_CSHARP_CLASS_CACHE[cache_key] classes: dict[str, list[dict]] = {} @@ -3746,12 +3756,11 @@ def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: # scanned millions of paths and effectively hung. A real .NET project sits # well under the cap; a runaway root is bounded to a fast, partial scan # instead of hanging. - import os as _os _DIR_CAP = 20000 cs_files: list[Path] = [] visited = 0 try: - for dirpath, dirnames, filenames in _os.walk(root): + for dirpath, dirnames, filenames in _walk_path(root): dirnames[:] = [ d for d in dirnames if not d.startswith(".") and not _is_noise_dir(d) ] @@ -3802,7 +3811,7 @@ def _xaml_communitytoolkit_members(vm_node: dict) -> tuple[dict[str, dict], list try: # errors="replace" so a non-UTF8 code-behind can't raise UnicodeDecodeError # and abort the whole extract_xaml (matches every other reader here). - lines = Path(source_file).read_text(encoding="utf-8", errors="replace").splitlines() + lines = _read_file_text(Path(source_file), encoding="utf-8", errors="replace").splitlines() except OSError: return {}, [] @@ -3866,7 +3875,7 @@ def extract_xaml(path: Path) -> dict: import xml.etree.ElementTree as ET try: - src = path.read_bytes() + src = _read_file_bytes(path) except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -4280,7 +4289,7 @@ def _is_objc_header(path: Path) -> bool: extract_objc while leaving every C/C++ header on its existing extractor. """ try: - head = path.read_bytes()[:256 * 1024] + head = _read_file_bytes(path, limit=256 * 1024) except OSError: return False return any(marker in head for marker in _OBJC_HEADER_MARKERS) @@ -4323,7 +4332,7 @@ def _is_cpp_header(path: Path) -> bool: here and keeps its existing extract_c routing. """ try: - head = path.read_bytes()[:256 * 1024] + head = _read_file_bytes(path, limit=256 * 1024) except OSError: return False return any(marker in head for marker in _CPP_HEADER_MARKERS) @@ -4378,7 +4387,7 @@ def _get_extractor(path: Path) -> Any | None: def _safe_extract_with_xaml_root(extractor, path: Path, root: Path) -> dict: global _XAML_ACTIVE_EXTRACT_ROOT previous_root = _XAML_ACTIVE_EXTRACT_ROOT - _XAML_ACTIVE_EXTRACT_ROOT = root.resolve() + _XAML_ACTIVE_EXTRACT_ROOT = _resolve_path(root) try: return _safe_extract(extractor, path) finally: @@ -4647,7 +4656,7 @@ def extract( root = anchor_root elif cache_root is not None: root = cache_root - root = root.resolve() + root = _resolve_path(root) # #1774: the cache is an OUTPUT, so when no explicit cache_root is given it is # written under the current working directory — never `root` (the inferred @@ -4655,7 +4664,7 @@ def extract( # read-only or foreign corpus. `root` still anchors the content-hash keys, # node ids, symbol resolution, and the XAML project-scan boundary; only the # cache directory's location diverges from it. - cache_location = (cache_root if cache_root is not None else Path(".")).resolve() + cache_location = _resolve_path(cache_root if cache_root is not None else Path(".")) total = len(paths) # Phase 1: separate cached hits from uncached work @@ -4847,7 +4856,7 @@ def _portable_out_of_root_sf(p: Path) -> str: _remap_seen: set[Path] = set() for _p in paths: try: - _remap_seen.add(_p.resolve()) + _remap_seen.add(_resolve_path(_p)) except (OSError, RuntimeError): pass for _e in all_edges: @@ -4856,7 +4865,7 @@ def _portable_out_of_root_sf(p: Path) -> str: continue _raw_tp = Path(_tf) try: - _tp = _raw_tp.resolve() + _tp = _resolve_path(_raw_tp) except (OSError, RuntimeError): continue if _tp in _remap_seen: @@ -4882,7 +4891,7 @@ def _portable_out_of_root_sf(p: Path) -> str: # target that does not actually exist on disk stays dangling, # exactly as before. try: - if _tp.is_file(): + if _path_is_file(_tp): ext_new_id = _make_id("ext", _portable_out_of_root_sf(_tp)) id_remap[_make_id(str(_tp))] = ext_new_id if _raw_tp != _tp: @@ -4902,7 +4911,7 @@ def _portable_out_of_root_sf(p: Path) -> str: pass continue try: - if not _tp.is_file(): + if not _path_is_file(_tp): # Speculatively-resolved target that doesn't exist (e.g. an # import of a not-yet-created sibling): keep its raw id # dangling, exactly as before, so no false canonical edge is @@ -4927,7 +4936,7 @@ def _portable_out_of_root_sf(p: Path) -> str: rel = path.relative_to(root) except ValueError: try: - rel = path.resolve().relative_to(root) + rel = _resolve_path(path).relative_to(root) except ValueError: continue new_id = _file_node_id(rel) @@ -4936,14 +4945,14 @@ def _portable_out_of_root_sf(p: Path) -> str: # Also register the absolute-resolved form of the file-level id so # alias/workspace import targets (resolved via .resolve()) remap to # canonical instead of orphaning (#1529). - old_id_abs = _make_id(str(path.resolve())) + old_id_abs = _make_id(str(_resolve_path(path))) if old_id_abs != new_id: id_remap[old_id_abs] = new_id old_prefs: list[tuple[str, str]] = [] old_pref = _file_node_id(path) if old_pref != new_id: old_prefs.append((old_pref, new_id)) - old_pref_abs = _file_node_id(path.resolve()) + old_pref_abs = _file_node_id(_resolve_path(path)) if old_pref_abs != new_id and old_pref_abs != old_pref: old_prefs.append((old_pref_abs, new_id)) # Bash entrypoint node ids append "__entry" to the file-level id @@ -4962,10 +4971,10 @@ def _portable_out_of_root_sf(p: Path) -> str: if _entry_old != _entry_new: id_remap.setdefault(_entry_old, _entry_new) if old_prefs: - prefix_remap[path.resolve()] = old_prefs + prefix_remap[_resolve_path(path)] = old_prefs # Absolute form first: it is the longest, so prefix decomposition can # try forms in order without a shorter form shadowing it. - stem_forms[path.resolve()] = ( + stem_forms[_resolve_path(path)] = ( new_id, [old_pref_abs, old_pref, new_id] ) if id_remap: @@ -5003,7 +5012,7 @@ def _portable_out_of_root_sf(p: Path) -> str: if n.get("type") == "package": continue try: - entry = prefix_remap.get(Path(sf).resolve()) + entry = prefix_remap.get(_resolve_path(sf)) except Exception: continue if entry is None: @@ -5102,7 +5111,7 @@ def _edge_key(edge: dict) -> str: def _decompose(target: str, tf: str) -> "tuple[str, str] | None": try: - forms = stem_forms.get(Path(tf).resolve()) + forms = stem_forms.get(_resolve_path(tf)) except (OSError, RuntimeError): return None if not forms: @@ -5612,7 +5621,7 @@ def _sf_entry(sf: str, sf_path: Path) -> tuple[str, str, tuple[str, ...]]: canonical_id = _file_node_id(rel) new_sf = rel.as_posix() try: - sf_resolved = sf_path.resolve() + sf_resolved = _resolve_path(sf_path) except (OSError, RuntimeError): sf_resolved = sf_path # Learn the STEM (extension-dropped) forms too: symbol producers mint @@ -5733,7 +5742,7 @@ def _canon(nid: str) -> str: def collect_files(target: Path, *, follow_symlinks: bool = False, root: Path | None = None) -> list[Path]: containment_root = root if root is not None else target from graphify.detect import _resolves_under_root - if target.is_file(): + if _path_is_file(target): return [target] if _resolves_under_root(target, containment_root) else [] _EXTENSIONS = set(_DISPATCH.keys()) from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore @@ -5756,7 +5765,7 @@ def _ignored(p: Path) -> bool: # conservatism as detect's scan walk). has_negation = any(pat.startswith("!") for _, pat in patterns) results: list[Path] = [] - for dirpath, dirnames, filenames in os.walk(target): + for dirpath, dirnames, filenames in _walk_path(target): dp = Path(dirpath) dirnames[:] = [ d for d in dirnames @@ -5771,10 +5780,10 @@ def _ignored(p: Path) -> bool: return sorted(results) # Walk with symlink following + cycle detection results = [] - for dirpath, dirnames, filenames in os.walk(target, followlinks=True): - if os.path.islink(dirpath): - real = os.path.realpath(dirpath) - parent_real = os.path.realpath(os.path.dirname(dirpath)) + for dirpath, dirnames, filenames in _walk_path(target, followlinks=True): + if _path_is_symlink(dirpath): + real = str(_resolve_path(dirpath)) + parent_real = str(_resolve_path(Path(dirpath).parent)) if parent_real == real or parent_real.startswith(real + os.sep): dirnames.clear() continue @@ -5782,7 +5791,7 @@ def _ignored(p: Path) -> bool: dirnames[:] = [ d for d in dirnames if not _is_noise_dir(d, dp) # pass parent so "env"/"*_env" is marker-gated (#2058) - and (not (dp / d).is_symlink() or _resolves_under_root(dp / d, containment_root)) + and (not _path_is_symlink(dp / d) or _resolves_under_root(dp / d, containment_root)) ] for fname in filenames: p = dp / fname diff --git a/graphify/extractors/apex.py b/graphify/extractors/apex.py index 928923a64..578ea4c49 100644 --- a/graphify/extractors/apex.py +++ b/graphify/extractors/apex.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _file_stem, _make_id @@ -11,7 +12,7 @@ def extract_apex(path: Path) -> dict: Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI).""" import re as _re try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": []} diff --git a/graphify/extractors/bash.py b/graphify/extractors/bash.py index 984676b2a..0923329d0 100644 --- a/graphify/extractors/bash.py +++ b/graphify/extractors/bash.py @@ -6,6 +6,12 @@ from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + path_exists as _path_exists, + path_is_file as _path_is_file, + read_bytes as _read_file_bytes, + resolve_path as _resolve_path, +) # Leading `${VAR}` / `$VAR` expansion segment(s) of a `source` path argument. The @@ -79,7 +85,7 @@ def extract_bash(path: Path) -> dict: try: language = Language(tsbash.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -266,12 +272,12 @@ def walk(node, parent_nid: str) -> None: raw = _read_text(args[0], source).strip().strip("'\"") line = node.start_point[0] + 1 if raw.startswith((".", "/")): - resolved = (path.parent / raw).resolve() + resolved = _resolve_path(path.parent / raw) # Only emit the edge if the target actually exists on # disk — prevents graph pollution from crafted paths # like `source ../../etc/passwd` that traverse outside # the project tree (B-1). - if resolved.exists(): + if _path_exists(resolved): tgt_nid = _make_id(str(resolved)) add_edge(file_nid, tgt_nid, "imports_from", line, context="import", @@ -306,8 +312,8 @@ def walk(node, parent_nid: str) -> None: var_name = var_match.group(1) or var_match.group(2) if var_name in var_bases: base = var_bases[var_name] - resolved = (base / suffix).resolve() - if resolved.is_file(): + resolved = _resolve_path(base / suffix) + if _path_is_file(resolved): add_edge(file_nid, _make_id(str(resolved)), "imports_from", line, confidence="INFERRED", context="import", @@ -335,8 +341,8 @@ def walk(node, parent_nid: str) -> None: if raw: try: candidate = path.parent / raw - if candidate.is_file(): - sibling = candidate.resolve() + if _path_is_file(candidate): + sibling = _resolve_path(candidate) except OSError: sibling = None if sibling is not None: @@ -362,12 +368,12 @@ def walk(node, parent_nid: str) -> None: if cmd in _BASH_SCRIPT_RUNNERS and args: raw = literal(args[0]) if raw and raw.endswith(".sh"): - resolved = (path.parent / raw).resolve() - if resolved.is_file(): + resolved = _resolve_path(path.parent / raw) + if _path_is_file(resolved): target_path = resolved if not path.is_absolute(): try: - target_path = resolved.relative_to(Path.cwd().resolve()) + target_path = resolved.relative_to(_resolve_path(Path.cwd())) except ValueError: pass caller_nid = entry_nid if parent_nid == file_nid else parent_nid diff --git a/graphify/extractors/blade.py b/graphify/extractors/blade.py index 63e4fac9a..92b466ae9 100644 --- a/graphify/extractors/blade.py +++ b/graphify/extractors/blade.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _make_id @@ -10,7 +11,7 @@ def extract_blade(path: Path) -> dict: """Extract @include, components, and wire:click bindings from Blade templates.""" import re try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"error": f"cannot read {path}"} diff --git a/graphify/extractors/dart.py b/graphify/extractors/dart.py index acbe19583..324d6c250 100644 --- a/graphify/extractors/dart.py +++ b/graphify/extractors/dart.py @@ -4,13 +4,19 @@ import re from pathlib import Path + from graphify.extractors.base import _file_stem, _make_id +from graphify.paths import ( + path_exists as _path_exists, + read_text as _read_file_text, + resolve_path as _resolve_path, +) def extract_dart(path: Path) -> dict: """Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"error": f"cannot read {path}"} @@ -40,8 +46,8 @@ def _comment_replace(match: re.Match) -> str: parent_ref = part_of_match.group(1) if parent_ref.endswith(".dart"): try: - parent_path = (path.parent / parent_ref).resolve() - if parent_path.exists(): + parent_path = _resolve_path(path.parent / parent_ref) + if _path_exists(parent_path): stem = _file_stem(parent_path) file_nid = _make_id(str(parent_path)) is_part = True diff --git a/graphify/extractors/dm.py b/graphify/extractors/dm.py index 7961022cb..24571517c 100644 --- a/graphify/extractors/dm.py +++ b/graphify/extractors/dm.py @@ -5,7 +5,15 @@ from pathlib import Path from typing import Any + from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + path_exists as _path_exists, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) def extract_dm(path: Path) -> dict: @@ -18,7 +26,7 @@ def extract_dm(path: Path) -> dict: try: language = Language(tsdm.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -87,18 +95,18 @@ def walk(node, parent_type_path: "str | None" = None, raw = _read_include_path(file_node) if raw: norm = re.sub(r"^\./", "", raw.replace("\\", "/")) - resolved = (path.parent / norm).resolve() + resolved = _resolve_path(path.parent / norm) edge: dict = { "source": file_nid, - "target": _make_id(str(resolved)) if resolved.exists() else _make_id(norm), - "relation": "imports_from" if resolved.exists() else "imports", + "target": _make_id(str(resolved)) if _path_exists(resolved) else _make_id(norm), + "relation": "imports_from" if _path_exists(resolved) else "imports", "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, } - if not resolved.exists(): + if not _path_exists(resolved): edge["external"] = True edges.append(edge) return @@ -275,7 +283,7 @@ def _read_dmi_description(data: bytes) -> str: def extract_dmi(path: Path) -> dict: """Extract icon state names from a .dmi (BYOND PNG icon sheet).""" try: - data = path.read_bytes() + data = _read_file_bytes(path) except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -364,9 +372,9 @@ def _dmm_type_path(entry: str) -> str: def extract_dmm(path: Path) -> dict: """Extract type-path references from a .dmm map file's tile dictionary.""" try: - if path.stat().st_size > 50 * 1024 * 1024: + if _path_stat(path).st_size > 50 * 1024 * 1024: return {"nodes": [], "edges": [], "error": "file too large (>50 MB)"} - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -436,7 +444,7 @@ def extract_dmm(path: Path) -> dict: def extract_dmf(path: Path) -> dict: """Extract windows and controls from a .dmf interface file.""" try: - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/graphify/extractors/elixir.py b/graphify/extractors/elixir.py index 0f1588ddb..9bd2d7ee0 100644 --- a/graphify/extractors/elixir.py +++ b/graphify/extractors/elixir.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id @@ -18,7 +19,7 @@ def extract_elixir(path: Path) -> dict: try: language = Language(tselixir.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index e16d1fc78..85b89441a 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -9,6 +9,7 @@ from graphify.extractors.resolution import _resolve_js_import_target from graphify.security import sanitize_metadata from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes def _csharp_namespace_id(dotted_name: str) -> str: @@ -2314,7 +2315,7 @@ def _extract_generic( try: parser = Parser(language) - source = path.read_bytes() if source_override is None else source_override + source = _read_file_bytes(path) if source_override is None else source_override tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/fortran.py b/graphify/extractors/fortran.py index 58ffedfc8..6d03e1d7f 100644 --- a/graphify/extractors/fortran.py +++ b/graphify/extractors/fortran.py @@ -1,9 +1,13 @@ """Fortran extractor. Moved verbatim from graphify/extract.py.""" from __future__ import annotations - from pathlib import Path + from graphify.extractors.base import _file_stem, _make_id, _read_text +from graphify.paths import ( + read_bytes as _read_file_bytes, + resolve_path as _resolve_path, +) _FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"} @@ -25,13 +29,21 @@ def _cpp_preprocess(path: Path) -> bytes: import shutil import subprocess if not shutil.which("cpp"): - return path.read_bytes() + return _read_file_bytes(path) try: # Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot # be parsed by cpp as an option (cpp does not accept a "--" end-of-options # terminator). An absolute path always begins with "/". result = subprocess.run( - ["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())], + [ + "cpp", + "-w", + "-P", + "-nostdinc", + "-I", + "/dev/null", + str(_resolve_path(path)), + ], capture_output=True, timeout=30, ) @@ -39,7 +51,7 @@ def _cpp_preprocess(path: Path) -> bytes: return result.stdout except Exception: pass - return path.read_bytes() + return _read_file_bytes(path) def extract_fortran(path: Path) -> dict: """Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files. @@ -56,7 +68,11 @@ def extract_fortran(path: Path) -> dict: try: language = Language(tsfortran.language()) parser = Parser(language) - source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes() + source = ( + _cpp_preprocess(path) + if path.suffix in _FORTRAN_CPP_EXTS + else _read_file_bytes(path) + ) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/go.py b/graphify/extractors/go.py index d5654a5f0..ee5216910 100644 --- a/graphify/extractors/go.py +++ b/graphify/extractors/go.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text @@ -90,7 +91,7 @@ def extract_go(path: Path) -> dict: try: language = Language(tsgo.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/json_config.py b/graphify/extractors/json_config.py index 6a9b641a9..4c8e7d956 100644 --- a/graphify/extractors/json_config.py +++ b/graphify/extractors/json_config.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id, _read_text from graphify.ids import normalize_id @@ -68,8 +69,7 @@ def extract_json(path: Path) -> dict: # Bounded read instead of stat()+read() to eliminate TOCTOU (J-1): # read one byte beyond the limit so we can detect oversized files even # if the file grows between stat and read. - with path.open("rb") as _f: - source = _f.read(_JSON_MAX_BYTES + 1) + source = _read_file_bytes(path, limit=_JSON_MAX_BYTES + 1) if len(source) > _JSON_MAX_BYTES: return {"nodes": [], "edges": [], "error": "json file too large to index"} language = Language(tsjson.language()) diff --git a/graphify/extractors/julia.py b/graphify/extractors/julia.py index 95846c7d9..a9ae03794 100644 --- a/graphify/extractors/julia.py +++ b/graphify/extractors/julia.py @@ -4,6 +4,7 @@ from graphify.extractors.base import _file_stem, _make_id, _read_text from graphify.extractors.engine import _semantic_reference_edge from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes def extract_julia(path: Path) -> dict: @@ -17,7 +18,7 @@ def extract_julia(path: Path) -> dict: try: language = Language(tsjulia.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/markdown.py b/graphify/extractors/markdown.py index e1b24409a..d28e92658 100644 --- a/graphify/extractors/markdown.py +++ b/graphify/extractors/markdown.py @@ -5,6 +5,7 @@ import os from pathlib import Path +from graphify.paths import read_text as _read_file_text, path_is_file as _path_is_file from graphify.extractors.base import _file_stem, _make_id @@ -77,7 +78,7 @@ def extract_markdown(path: Path) -> dict: No tree-sitter dependency — pure line-by-line parsing. """ try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -134,7 +135,7 @@ def add_link(raw: str, line: int) -> None: # and popped before graph.json ships. target_file = None try: - if resolved.is_file(): + if _path_is_file(resolved): target_file = str(resolved) except OSError: pass diff --git a/graphify/extractors/objc.py b/graphify/extractors/objc.py index 9f978a50f..3884e3dfd 100644 --- a/graphify/extractors/objc.py +++ b/graphify/extractors/objc.py @@ -5,6 +5,7 @@ from graphify.extractors.engine import _cpp_declarator_name, _semantic_reference_edge from graphify.extractors.resolution import _resolve_c_include_path from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any @@ -51,7 +52,7 @@ def extract_objc(path: Path) -> dict: try: language = Language(tsobjc.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) # tree-sitter-objc cannot expand these argument-less annotation macros (no # trailing ';'), and their presence before @interface makes the parser fail to # emit a class_interface node (#1475). Blank them to equal-length spaces so byte diff --git a/graphify/extractors/pascal.py b/graphify/extractors/pascal.py index 398edb22e..c8e8e78ab 100644 --- a/graphify/extractors/pascal.py +++ b/graphify/extractors/pascal.py @@ -5,6 +5,7 @@ from graphify.extractors.base import _file_stem, _make_id from graphify.extractors.resolution import _pascal_resolve_class, _pascal_resolve_unit from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes, read_text as _read_file_text from typing import Any, Callable @@ -233,7 +234,7 @@ def _extract_pascal_regex(path: Path) -> dict: is unavailable. Produces the same node/edge schema as the tree-sitter pass. """ try: - raw = path.read_text(encoding="utf-8", errors="replace") + raw = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as exc: return {"nodes": [], "edges": [], "error": str(exc)} @@ -457,7 +458,7 @@ def extract_pascal(path: Path) -> dict: try: language = Language(tspascal.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception: diff --git a/graphify/extractors/pascal_forms.py b/graphify/extractors/pascal_forms.py index 1f81ad875..9cd09f4ff 100644 --- a/graphify/extractors/pascal_forms.py +++ b/graphify/extractors/pascal_forms.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes, read_text as _read_file_text from typing import Any from graphify.extractors.base import _file_stem, _make_id @@ -30,7 +31,7 @@ def extract_lazarus_form(path: Path) -> dict: - component --references--> event handler (context: "event") """ try: - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -116,7 +117,7 @@ def extract_delphi_form(path: Path) -> dict: (`contains`) and event handler references (`references`, context "event"). """ try: - raw = path.read_bytes() + raw = _read_file_bytes(path) except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/graphify/extractors/powershell.py b/graphify/extractors/powershell.py index 9f8526c7b..2345a9d9a 100644 --- a/graphify/extractors/powershell.py +++ b/graphify/extractors/powershell.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -19,7 +20,7 @@ def extract_powershell(path: Path) -> dict: try: language = Language(tsps.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -370,7 +371,7 @@ def extract_powershell_manifest(path: Path) -> dict: try: language = Language(tsps.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/razor.py b/graphify/extractors/razor.py index 09dc54a6b..362031e44 100644 --- a/graphify/extractors/razor.py +++ b/graphify/extractors/razor.py @@ -3,6 +3,7 @@ import re from pathlib import Path +from graphify.paths import read_text as _read_file_text from graphify.extractors.base import _file_stem, _make_id @@ -10,7 +11,7 @@ def extract_razor(path: Path) -> dict: """Extract directives, component refs, and @code methods from .razor/.cshtml.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 097c32b6a..f3569c2aa 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -1,8 +1,19 @@ """resolution — moved verbatim from graphify/extract.py.""" from __future__ import annotations -from typing import Any, Callable from pathlib import Path +from typing import Any, Callable + +from graphify.paths import ( + glob_paths as _glob_paths, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 from graphify.extractors.base import ( # noqa: F401 _LANGUAGE_BUILTIN_GLOBALS, @@ -31,31 +42,31 @@ def _resolve_js_import_path(candidate: Path) -> Path: """Resolve a JS/TS/Svelte import target to a local file when it exists.""" candidate = Path(os.path.normpath(candidate)) - if candidate.is_file(): + if _path_is_file(candidate): return candidate # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. if candidate.suffix == ".js": ts_candidate = candidate.with_suffix(".ts") - if ts_candidate.is_file(): + if _path_is_file(ts_candidate): return ts_candidate elif candidate.suffix == ".jsx": tsx_candidate = candidate.with_suffix(".tsx") - if tsx_candidate.is_file(): + if _path_is_file(tsx_candidate): return tsx_candidate # Append extensions to the full filename, which covers extensionless imports, # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. for ext in _JS_RESOLVE_EXTS: with_ext = candidate.parent / f"{candidate.name}{ext}" - if with_ext.is_file(): + if _path_is_file(with_ext): return with_ext # Only fall back to directory indexes after file candidates lose. - if candidate.is_dir(): + if _path_is_dir(candidate): for index_name in _JS_INDEX_FILES: index_candidate = candidate / index_name - if index_candidate.is_file(): + if _path_is_file(index_candidate): return index_candidate return candidate @@ -98,7 +109,7 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st return {} seen.add(str(tsconfig)) try: - raw = tsconfig.read_text(encoding="utf-8") + raw = _read_file_text(tsconfig, encoding="utf-8") except Exception as e: print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) return {} @@ -132,10 +143,10 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. if not ext or ext.startswith("@"): continue - extended_path = (base_dir / ext).resolve() + extended_path = _resolve_path(base_dir / ext) if not extended_path.suffix: extended_path = extended_path.with_suffix(".json") - if extended_path.exists(): + if _path_exists(extended_path): aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen)) # tsconfig `paths` are resolved relative to `baseUrl` (itself relative to @@ -175,7 +186,7 @@ def _read_json_config(path: Path) -> "dict | None": baseUrl" instead of raising. """ try: - raw = path.read_text(encoding="utf-8", errors="replace") + raw = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return None for candidate in (raw, _strip_jsonc(raw)): @@ -195,11 +206,11 @@ def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": tsconfig.json wins when both sit in one directory, matching tsc and editors, which consult jsconfig.json only when there is no tsconfig.json. """ - current = start_dir.resolve() + current = _resolve_path(start_dir) for candidate in [current, *current.parents]: for name in ("tsconfig.json", "jsconfig.json"): config = candidate / name - if config.exists(): + if _path_exists(config): return config, candidate return None @@ -299,7 +310,7 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], if base_url is not None: candidate = Path(os.path.normpath(base_url / raw)) resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): + if _path_is_file(resolved): return resolved return None @@ -315,21 +326,21 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], if captured: cand = Path(os.path.normpath(cand / captured)) resolved = _resolve_js_import_path(cand) - if resolved.is_file(): + if _path_is_file(resolved): return resolved if first is None: first = cand return first def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() + current = _resolve_path(start_dir) for candidate in [current, *current.parents]: - if (candidate / "pnpm-workspace.yaml").exists(): + if _path_exists(candidate / "pnpm-workspace.yaml"): return candidate package_json = candidate / "package.json" - if package_json.is_file(): + if _path_is_file(package_json): try: - data = json.loads(package_json.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(package_json, encoding="utf-8")) except Exception: continue if "workspaces" in data: @@ -339,7 +350,8 @@ def _find_workspace_root(start_dir: Path) -> Path | None: def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: globs: list[str] = [] in_packages = False - for raw_line in workspace_file.read_text(encoding="utf-8", errors="replace").splitlines(): + text = _read_file_text(workspace_file, encoding="utf-8", errors="replace") + for raw_line in text.splitlines(): line = raw_line.strip() if not line or line.startswith("#"): continue @@ -357,12 +369,12 @@ def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: def _workspace_globs(root: Path) -> list[str]: pnpm_workspace = root / "pnpm-workspace.yaml" - if pnpm_workspace.exists(): + if _path_exists(pnpm_workspace): return _pnpm_workspace_globs(pnpm_workspace) package_json = root / "package.json" try: - data = json.loads(package_json.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(package_json, encoding="utf-8")) except Exception: return [] @@ -380,9 +392,9 @@ def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: if root is None: return {} manifest_mtimes = tuple( - (name, (root / name).stat().st_mtime_ns) + (name, _path_stat(root / name).st_mtime_ns) for name in _WORKSPACE_MANIFEST_NAMES - if (root / name).is_file() + if _path_is_file(root / name) ) key = str((root, manifest_mtimes)) if key in _WORKSPACE_PACKAGE_CACHE: @@ -390,13 +402,15 @@ def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: packages: dict[str, Path] = {} for pattern in _workspace_globs(root): - package_dirs: list[Path] = [root] if pattern in (".", "./") else list(root.glob(pattern)) + package_dirs: list[Path] = ( + [root] if pattern in (".", "./") else list(_glob_paths(root, pattern)) + ) for package_dir in package_dirs: manifest = package_dir / "package.json" - if not manifest.is_file(): + if not _path_is_file(manifest): continue try: - data = json.loads(manifest.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(manifest, encoding="utf-8")) except Exception: continue name = data.get("name") @@ -431,7 +445,7 @@ def _contained_in_package(resolved: Path, package_dir: Path) -> bool: (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay within package_dir after resolution.""" try: - return resolved.resolve().is_relative_to(package_dir.resolve()) + return _resolve_path(resolved).is_relative_to(_resolve_path(package_dir)) except ValueError: return False @@ -439,7 +453,7 @@ def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: manifest = package_dir / "package.json" manifest_data: dict[str, Any] = {} try: - manifest_data = json.loads(manifest.read_text(encoding="utf-8")) + manifest_data = json.loads(_read_file_text(manifest, encoding="utf-8")) except Exception: pass @@ -498,7 +512,7 @@ def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: continue for candidate in _package_entry_candidates(package_dir, subpath): resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): + if _path_is_file(resolved): return resolved return None @@ -560,8 +574,8 @@ def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": """ if not raw: return None - candidate = (Path(str_path).parent / raw).resolve() - if candidate.is_file(): + candidate = _resolve_path(Path(str_path).parent / raw) + if _path_is_file(candidate): return candidate return None @@ -591,11 +605,11 @@ def _resolve_lua_import_target(raw_module: str, str_path: str) -> str: for _ in range(6): for suffix in (".lua", ".luau"): cand = probe / f"{rel}{suffix}" - if cand.is_file(): + if _path_is_file(cand): return _make_id(str(cand)) for suffix in (".lua", ".luau"): cand = probe / rel / f"init{suffix}" - if cand.is_file(): + if _path_is_file(cand): return _make_id(str(cand)) if probe.parent == probe: break @@ -642,7 +656,7 @@ def _source_key(source_file: str, root: Path) -> str: return "" source_path = Path(source_file) try: - return str(source_path.resolve().relative_to(root)) + return str(_resolve_path(source_path).relative_to(_resolve_path(root))) except Exception: return str(source_path) @@ -812,7 +826,7 @@ def _js_source_path(source_file: str, root: Path) -> Path | None: if not path.is_absolute(): path = root / path try: - return path.resolve() + return _resolve_path(path) except Exception: return path @@ -836,8 +850,8 @@ def _apply_symbol_resolution_facts( ): return - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + path_by_resolved = {_resolve_path(path): path for path in paths} + source_file_id = {_resolve_path(path): _make_id(str(path)) for path in paths} symbol_nodes: dict[tuple[Path, str], str] = {} for node in nodes: source_path = _js_source_path(str(node.get("source_file", "")), root) @@ -848,7 +862,7 @@ def _apply_symbol_resolution_facts( symbol_nodes[(source_path, label)] = str(node["id"]) def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) existing = symbol_nodes.get((resolved_path, name)) if existing is not None: return existing @@ -905,15 +919,16 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() + file_path = _resolve_path(import_fact.file_path) local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), + _resolve_path(import_fact.target_path), import_fact.imported_name, ) pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + resolved_file = _resolve_path(alias_fact.file_path) + pending_aliases_by_file.setdefault(resolved_file, []).append(alias_fact) for file_path, aliases in pending_aliases_by_file.items(): local_aliases = local_aliases_by_file.setdefault(file_path, {}) @@ -932,8 +947,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() + source_path = _resolve_path(star_fact.file_path) + target_path = _resolve_path(star_fact.target_path) star_exports_by_file.setdefault(source_path, []).append(target_path) source_id = source_file_id.get(source_path) if source_id is not None: @@ -948,8 +963,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() + source_path = _resolve_path(namespace_fact.file_path) + target_path = _resolve_path(namespace_fact.target_path) namespace_id = ensure_symbol_node( namespace_fact.file_path, namespace_fact.exported_name, @@ -979,10 +994,10 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() + file_path = _resolve_path(export_fact.file_path) origin: tuple[Path, str] | None = None if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) + origin = (_resolve_path(export_fact.target_path), export_fact.target_name) elif export_fact.local_name is not None: origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) if origin is None and (file_path, export_fact.local_name) in symbol_nodes: @@ -1004,7 +1019,7 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() + target_path = _resolve_path(target_path) key = (target_path, imported_name) if seen is None: seen = set() @@ -1024,7 +1039,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup return key for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) + source_id = source_file_id.get(_resolve_path(import_fact.file_path)) if source_id is None: continue origin_path, origin_symbol = resolve_exported_origin( @@ -1068,7 +1083,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup # canonicalizes — or drop it when no file node id is available. owned = {str(n.get("id")) for n in nodes} for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() + file_path = _resolve_path(use_fact.file_path) target_id = None unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) if unresolved_origin is not None: @@ -1106,11 +1121,11 @@ def _parse_js_tree(path: Path): vue_lang: str | None = None if path.suffix == ".vue": masked, vue_lang = _vue_mask_non_script( - path.read_text(encoding="utf-8", errors="replace") + _read_file_text(path, encoding="utf-8", errors="replace") ) source = masked.encode("utf-8") else: - source = path.read_bytes() + source = _read_file_bytes(path) use_ts = path.suffix in (".ts", ".mts", ".cts") or ( path.suffix == ".vue" and vue_lang not in ("js", "jsx") ) @@ -1475,7 +1490,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut trees: dict[Path, tuple[bytes, object]] = {} for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = _parse_js_tree(path) if parsed is None: continue @@ -1497,7 +1512,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_path(target_path) for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"): facts.imports.append( _SymbolImportFact( @@ -1527,7 +1542,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1543,7 +1558,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = _resolve_path(target_path) namespace_name = _js_namespace_export_name(node, source) if namespace_name is not None: facts.namespace_exports.append( @@ -1613,7 +1628,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1635,7 +1650,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = _resolve_path(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1661,7 +1676,7 @@ def _parse_python_tree(path: Path): try: import tree_sitter_python as tspython from tree_sitter import Language, Parser - source = path.read_bytes() + source = _read_file_bytes(path) parser = Parser(Language(tspython.language())) return source, parser.parse(source).root_node except Exception: @@ -1718,14 +1733,14 @@ def _python_imported_names(node, source: bytes) -> list[tuple[str, str]]: def _probe_python_module_candidate(candidate: Path) -> Path | None: """Resolve one module-path candidate to a .py file (dir+__init__, exact, or with a .py suffix), or None.""" - if candidate.is_dir(): + if _path_is_dir(candidate): init_path = candidate / "__init__.py" - if init_path.is_file(): + if _path_is_file(init_path): return init_path - if candidate.is_file(): + if _path_is_file(candidate): return candidate py_candidate = candidate.with_suffix(".py") - if py_candidate.is_file(): + if _path_is_file(py_candidate): return py_candidate return None @@ -1761,7 +1776,7 @@ def _resolve_python_module_path(module_name: str, current_path: Path, root: Path # implicit-relative semantics), fabricating edges to what may be an # external dependency (#2072 review). A src-layout root (src/, no # __init__.py) is still probed. - if (anc / "__init__.py").is_file(): + if _path_is_file(anc / "__init__.py"): continue cand = _probe_python_module_candidate(anc / rel) if cand is not None: @@ -1803,7 +1818,7 @@ def _collect_python_symbol_resolution_facts( if parsed is None: continue source, root_node = parsed - trees[path.resolve()] = parsed + trees[_resolve_path(path)] = parsed for node in _walk_python_tree(root_node): if node.type != "import_from_statement": @@ -1825,7 +1840,11 @@ def _collect_python_symbol_resolution_facts( if pkg_dir is not None: sub_py = pkg_dir / f"{imported_name}.py" sub_pkg = pkg_dir / imported_name / "__init__.py" - submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) + submodule = ( + sub_py + if _path_is_file(sub_py) + else (sub_pkg if _path_is_file(sub_pkg) else None) + ) if submodule is not None: facts.module_imports.append((path, submodule, line, local_name)) continue @@ -1844,7 +1863,7 @@ def _collect_python_symbol_resolution_facts( ) for path in py_paths: - parsed = trees.get(path.resolve()) + parsed = trees.get(_resolve_path(path)) if parsed is None: continue source, root_node = parsed @@ -1956,7 +1975,7 @@ def _resolve_cross_file_imports( # Parse imports from this file try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2208,7 +2227,7 @@ def _resolve_cross_file_java_imports( for path in paths: file_nid = _make_id(str(path)) try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2291,7 +2310,7 @@ def _resolve_java_type_references( if not srcs: continue try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2454,7 +2473,7 @@ def _resolve_php_type_references( if not srcs: continue try: - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) except Exception: continue @@ -2670,8 +2689,8 @@ def _pascal_project_root(from_path: Path) -> Path: for _ in range(12): if len(current.parts) <= 1: break # never use a filesystem root (D:/, C:/, /) - pas_count = sum(1 for _ in current.glob("*.pas")) - dpr_count = sum(1 for _ in current.glob("*.dpr")) + pas_count = sum(1 for _ in _glob_paths(current, "*.pas")) + dpr_count = sum(1 for _ in _glob_paths(current, "*.dpr")) if pas_count >= 2 or dpr_count >= 1: best = current parent = current.parent @@ -2694,7 +2713,7 @@ def _pascal_resolve_unit(from_path: Path, unit_name: str) -> str: if root_key not in _pascal_unit_cache: unit_map: dict[str, str] = {} for ext in (".pas", ".pp", ".dpr", ".dpk", ".inc"): - for f in root.rglob("*" + ext): + for f in _glob_paths(root, "**/*" + ext): unit_map[f.stem.lower()] = _make_id(str(f)) _pascal_unit_cache[root_key] = unit_map return _pascal_unit_cache[root_key].get(unit_name.lower(), _make_id(unit_name)) @@ -2717,7 +2736,7 @@ def _pascal_resolve_class(from_path: Path, class_name: str) -> str | None: if root_key not in _pascal_class_stem_cache: stem_map: dict[str, str] = {} for ext in (".pas", ".pp", ".dpr", ".dpk"): - for f in root.rglob("*" + ext): + for f in _glob_paths(root, "**/*" + ext): stem_map[f.stem.lower()] = _file_stem(f) _pascal_class_stem_cache[root_key] = stem_map diff --git a/graphify/extractors/rust.py b/graphify/extractors/rust.py index b663bd625..19e3927d2 100644 --- a/graphify/extractors/rust.py +++ b/graphify/extractors/rust.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text @@ -69,7 +70,7 @@ def extract_rust(path: Path) -> dict: try: language = Language(tsrust.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/sln.py b/graphify/extractors/sln.py index 936ff9fff..8d99c4a33 100644 --- a/graphify/extractors/sln.py +++ b/graphify/extractors/sln.py @@ -4,13 +4,14 @@ import re from pathlib import Path +from graphify.paths import read_text as _read_file_text, resolve_path as _resolve_path from graphify.extractors.base import _make_id def extract_sln(path: Path) -> dict: """Extract projects and inter-project dependencies from a .sln file.""" try: - src = path.read_text(encoding="utf-8", errors="replace") + src = _read_file_text(path, encoding="utf-8", errors="replace") except OSError: return {"nodes": [], "edges": [], "error": f"cannot read {path}"} @@ -46,7 +47,7 @@ def extract_sln(path: Path) -> dict: abs_proj = proj_name else: try: - abs_proj = str((path.parent / proj_path).resolve()) + abs_proj = str(_resolve_path(path.parent / proj_path)) except Exception: abs_proj = proj_path proj_nid = _make_id(abs_proj) diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index 4b52eaef8..b35af3ac9 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id @@ -41,7 +42,7 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: source = ( content.encode("utf-8") if isinstance(content, str) else content if content is not None - else path.read_bytes() + else _read_file_bytes(path) ) tree = parser.parse(source) root = tree.root_node diff --git a/graphify/extractors/terraform.py b/graphify/extractors/terraform.py index b5fb78f99..2065d584a 100644 --- a/graphify/extractors/terraform.py +++ b/graphify/extractors/terraform.py @@ -3,6 +3,7 @@ from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _make_id @@ -31,7 +32,7 @@ def extract_terraform(path: Path) -> dict: try: language = Language(tshcl.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/verilog.py b/graphify/extractors/verilog.py index 2f5fea49b..2de5df442 100644 --- a/graphify/extractors/verilog.py +++ b/graphify/extractors/verilog.py @@ -4,6 +4,7 @@ import re from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -215,7 +216,7 @@ def extract_verilog(path: Path) -> dict: try: language = Language(tsverilog.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/zig.py b/graphify/extractors/zig.py index 9744c514b..76bbc2d78 100644 --- a/graphify/extractors/zig.py +++ b/graphify/extractors/zig.py @@ -2,6 +2,7 @@ from __future__ import annotations from pathlib import Path +from graphify.paths import read_bytes as _read_file_bytes from typing import Any from graphify.extractors.base import _file_stem, _make_id, _read_text @@ -18,7 +19,7 @@ def extract_zig(path: Path) -> dict: try: language = Language(tszig.language()) parser = Parser(language) - source = path.read_bytes() + source = _read_file_bytes(path) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/file_slice.py b/graphify/file_slice.py index 30dc49cfb..0bb98986d 100644 --- a/graphify/file_slice.py +++ b/graphify/file_slice.py @@ -22,6 +22,8 @@ from dataclasses import dataclass from pathlib import Path +from graphify.paths import read_text as _read_file_text + # Plain-text document types where boundary-based slicing is meaningful and where # `_file_to_text` is a straight ``read_text`` (so a char range matches the bytes # the model is shown). Deliberately excludes code (.py, .ts, ...) and binary @@ -119,7 +121,7 @@ def expand_oversized_files( out.append(f) continue try: - text = f.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(f, encoding="utf-8", errors="replace") except OSError: out.append(f) continue @@ -135,7 +137,7 @@ def expand_oversized_files( def read_slice_text(fs: FileSlice) -> str: """Read just this slice's characters from its parent file.""" - text = fs.path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(fs.path, encoding="utf-8", errors="replace") return text[fs.start:fs.end] @@ -149,7 +151,7 @@ def bisect_slice(fs: FileSlice) -> tuple[FileSlice, FileSlice] | None: if fs.end - fs.start <= 1: return None try: - text = fs.path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(fs.path, encoding="utf-8", errors="replace") except OSError: return None mid = (fs.start + fs.end) // 2 diff --git a/graphify/google_workspace.py b/graphify/google_workspace.py index 1feeb8acd..c74d5e26e 100644 --- a/graphify/google_workspace.py +++ b/graphify/google_workspace.py @@ -18,6 +18,15 @@ from pathlib import Path from typing import Callable, Any +from graphify.paths import ( + io_path as _io_path, + make_dirs as _make_dirs, + read_text as _read_file_text, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_file_text, +) + GOOGLE_WORKSPACE_EXTENSIONS = {".gdoc", ".gsheet", ".gslides"} @@ -63,7 +72,7 @@ def _extract_resource_key(url: str, data: dict[str, Any]) -> str | None: def read_google_shortcut(path: Path) -> dict[str, str | None]: """Read a .gdoc/.gsheet/.gslides shortcut and return export metadata.""" try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(path, encoding="utf-8")) except Exception as exc: raise RuntimeError(f"could not read Google Workspace shortcut {path}: {exc}") from exc @@ -104,12 +113,14 @@ def _run_gws_export(file_id: str, mime_type: str, output: Path, resource_key: st # gws export command has no custom-header flag, so do not pass resourceKey # as an unsupported query parameter. _ = resource_key - output = output.resolve() - output.parent.mkdir(parents=True, exist_ok=True) + output = _resolve_path(output) + _make_dirs(output.parent, exist_ok=True) timeout = int(os.environ.get("GRAPHIFY_GOOGLE_WORKSPACE_TIMEOUT", "120")) result = subprocess.run( [exe, "drive", "files", "export", "--params", json.dumps(params), "-o", output.name], capture_output=True, + # Keep transport-only ``\\?\`` spelling inside Python filesystem + # calls; third-party executables should receive the ordinary path. cwd=output.parent, text=True, timeout=timeout, @@ -132,9 +143,9 @@ def _sidecar_path(path: Path, out_dir: Path, root: "Path | None" = None) -> Path if root is None: root = out_dir.parent.parent try: - key = path.resolve().relative_to(Path(root).resolve()).as_posix() + key = _resolve_path(path).relative_to(_resolve_path(root)).as_posix() except (ValueError, OSError): - key = str(path.resolve()) + key = str(_resolve_path(path)) name_hash = hashlib.sha256(unicodedata.normalize("NFC", key).encode()).hexdigest()[:8] return out_dir / f"{path.stem}_{name_hash}.md" @@ -177,39 +188,63 @@ def convert_google_workspace_file( return None shortcut = read_google_shortcut(path) - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) out_path = _sidecar_path(path, out_dir, root=root) if ext == ".gdoc": - with tempfile.NamedTemporaryFile("w+b", suffix=".md", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".md", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: - _run_gws_export(shortcut["file_id"] or "", "text/markdown", tmp_path, shortcut.get("resource_key")) - body = tmp_path.read_text(encoding="utf-8", errors="replace") + _run_gws_export( + shortcut["file_id"] or "", + "text/markdown", + tmp_path, + shortcut.get("resource_key"), + ) + body = _read_file_text(tmp_path, encoding="utf-8", errors="replace") finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text(_with_frontmatter(path, shortcut, body, "text/markdown"), encoding="utf-8") + _write_file_text( + out_path, + _with_frontmatter(path, shortcut, body, "text/markdown"), + encoding="utf-8", + ) return out_path if ext == ".gslides": - with tempfile.NamedTemporaryFile("w+b", suffix=".txt", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".txt", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: - _run_gws_export(shortcut["file_id"] or "", "text/plain", tmp_path, shortcut.get("resource_key")) - body = tmp_path.read_text(encoding="utf-8", errors="replace") + _run_gws_export( + shortcut["file_id"] or "", + "text/plain", + tmp_path, + shortcut.get("resource_key"), + ) + body = _read_file_text(tmp_path, encoding="utf-8", errors="replace") finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text(_with_frontmatter(path, shortcut, body, "text/plain"), encoding="utf-8") + _write_file_text( + out_path, + _with_frontmatter(path, shortcut, body, "text/plain"), + encoding="utf-8", + ) return out_path if ext == ".gsheet": if xlsx_to_markdown is None: raise RuntimeError("Google Sheets export requires the office extra: pip install graphifyy[office,google]") - with tempfile.NamedTemporaryFile("w+b", suffix=".xlsx", delete=False, dir=out_dir) as tmp: + with tempfile.NamedTemporaryFile( + "w+b", suffix=".xlsx", delete=False, dir=_io_path(out_dir) + ) as tmp: tmp_path = Path(tmp.name) try: _run_gws_export( @@ -220,10 +255,11 @@ def convert_google_workspace_file( ) body = xlsx_to_markdown(tmp_path) finally: - tmp_path.unlink(missing_ok=True) + _unlink_path(tmp_path, missing_ok=True) if not body.strip(): return None - out_path.write_text( + _write_file_text( + out_path, _with_frontmatter( path, shortcut, diff --git a/graphify/llm.py b/graphify/llm.py index 30d7a6d6f..24937684e 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -16,6 +16,14 @@ from dataclasses import dataclass, replace from pathlib import Path +from graphify.paths import ( + path_is_file as _path_is_file, + path_stat as _path_stat, + read_bytes as _read_file_bytes, + read_text as _read_file_text, + resolve_path as _resolve_path, +) + from graphify.file_slice import ( FileSlice, bisect_slice, @@ -269,7 +277,7 @@ def _load_custom_providers() -> dict[str, dict]: local_path = _custom_providers_path(global_=False) global_path = _custom_providers_path(global_=True) allow_local = os.environ.get("GRAPHIFY_ALLOW_LOCAL_PROVIDERS", "").strip().lower() in ("1", "true", "yes") - if local_path.is_file() and not allow_local: + if _path_is_file(local_path) and not allow_local: print( f"[graphify] WARNING: ignoring project-local {local_path} (custom providers control " "where your corpus and API key are sent). Set GRAPHIFY_ALLOW_LOCAL_PROVIDERS=1 to load it.", @@ -279,9 +287,9 @@ def _load_custom_providers() -> dict[str, dict]: providers: dict[str, dict] = {} paths = [local_path, global_path] if allow_local else [global_path] for path in paths: - if path.is_file(): + if _path_is_file(path): try: - data = json.loads(path.read_text(encoding="utf-8")) + data = json.loads(_read_file_text(path, encoding="utf-8")) if isinstance(data, dict): for name, cfg in data.items(): if not (isinstance(name, str) and isinstance(cfg, dict)): @@ -505,14 +513,14 @@ def _file_to_text(path: Path) -> str: if path.suffix.lower() == ".pdf": from graphify.detect import extract_pdf_text return extract_pdf_text(path) - return path.read_text(encoding="utf-8", errors="replace") + return _read_file_text(path, encoding="utf-8", errors="replace") def _resolve_under_root(path: Path, root: Path) -> Path | None: """Return the resolved path only when it stays inside ``root``.""" try: - resolved_root = root.resolve() - resolved_path = path.resolve() + resolved_root = _resolve_path(root) + resolved_path = _resolve_path(path) resolved_path.relative_to(resolved_root) except (OSError, RuntimeError, ValueError): return None @@ -698,7 +706,7 @@ def _bind_node_evidence(result: dict, text_units: "list[Path | FileSlice]", root if not p.is_absolute(): p = root / p try: - key = p.resolve() + key = _resolve_path(p) except (OSError, RuntimeError): continue src = source_by_path.get(key) @@ -819,7 +827,7 @@ def _build_image_refs(image_files: list[Path], root: Path, *, read_bytes: bool = raw: bytes | None = None if read_bytes: try: - raw = abs_path.read_bytes() + raw = _read_file_bytes(abs_path) except OSError as exc: print(f"[graphify] could not read image {rel}: {exc}", file=sys.stderr) raw = None @@ -1842,14 +1850,14 @@ def _estimate_file_tokens(unit: "Path | FileSlice") -> int: return _IMAGE_TOKEN_ESTIMATE if _TOKENIZER is None: try: - size = path.stat().st_size + size = _path_stat(path).st_size except OSError: return 0 chars = min(size, _FILE_CHAR_CAP) + _PER_FILE_OVERHEAD_CHARS return chars // _CHARS_PER_TOKEN try: - content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] + content = _read_file_text(path, encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] except OSError: return 0 return len(_TOKENIZER.encode(content, disallowed_special=())) + (_PER_FILE_OVERHEAD_CHARS // _CHARS_PER_TOKEN) @@ -2407,7 +2415,7 @@ def _resolve_against_root(value: "str | Path") -> Path: if not p.is_absolute(): p = root / p try: - return p.resolve() + return _resolve_path(p) except (OSError, RuntimeError): return p @@ -2418,7 +2426,7 @@ def _out_of_scope(item: dict) -> bool: if not sf: return False p = _resolve_against_root(sf) - return p.is_file() and p not in _dispatched_resolved + return _path_is_file(p) and p not in _dispatched_resolved dropped_ids: set = set() dropped_files: set[str] = set() @@ -2466,7 +2474,7 @@ def _out_of_scope(item: dict) -> bool: covered.add(p if p.is_absolute() else (root / p)) uncovered = sorted( p for p in dispatched - if p.resolve() not in {c.resolve() for c in covered} + if _resolve_path(p) not in {_resolve_path(c) for c in covered} ) merged["uncovered_files"] = [str(p) for p in uncovered] if uncovered: diff --git a/graphify/manifest_ingest.py b/graphify/manifest_ingest.py index ae3aa61fc..f6c712ad7 100644 --- a/graphify/manifest_ingest.py +++ b/graphify/manifest_ingest.py @@ -21,6 +21,7 @@ from typing import Any from graphify.ids import make_id +from graphify.paths import path_stat as _path_stat, read_text as _read_file_text __all__ = ["is_package_manifest_path", "extract_package_manifest", "PACKAGE_MANIFEST_NAMES"] @@ -51,9 +52,9 @@ def _pkg_id(name: str) -> str: def extract_package_manifest(path: Path) -> dict[str, Any]: """Parse a package manifest into a canonical package node + ``depends_on`` edges.""" try: - if path.stat().st_size > _MAX_MANIFEST_BYTES: + if _path_stat(path).st_size > _MAX_MANIFEST_BYTES: return {"nodes": [], "edges": [], "error": "manifest too large to index"} - text = path.read_text(encoding="utf-8", errors="replace") + text = _read_file_text(path, encoding="utf-8", errors="replace") except OSError as exc: return {"nodes": [], "edges": [], "error": f"manifest read error: {exc}"} diff --git a/graphify/mcp_ingest.py b/graphify/mcp_ingest.py index 152e4093f..6bc2d9ca5 100644 --- a/graphify/mcp_ingest.py +++ b/graphify/mcp_ingest.py @@ -64,6 +64,7 @@ from typing import Any from graphify.ids import make_id as _shared_make_id +from graphify.paths import read_bytes as _read_file_bytes from graphify.security import sanitize_label @@ -92,8 +93,7 @@ def extract_mcp_config(path: Path) -> dict[str, Any]: failure, oversize file, or missing ``mcpServers`` map """ try: - with path.open("rb") as fh: - raw = fh.read(_MAX_BYTES + 1) + raw = _read_file_bytes(path, limit=_MAX_BYTES + 1) except OSError as exc: return {"nodes": [], "edges": [], "error": f"mcp_ingest read error: {exc}"} diff --git a/graphify/paths.py b/graphify/paths.py index ef0cedb5b..84dae5671 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -1,31 +1,290 @@ -"""Single source of truth for the graphify output-directory name. +r"""Cross-platform filesystem boundaries and Graphify output paths. + +Graphify stores ordinary, user-facing paths in graph IDs, manifests, cache keys, +and diagnostics. On Windows, direct filesystem calls additionally need the +extended-length namespace (``\\?\`` for local paths and ``\\?\UNC\`` for +UNC paths) to reach deeply nested corpus files without depending on machine-wide +policy. The helpers in this module keep that transport-only spelling at the I/O +boundary so Linux and macOS remain no-ops and Windows paths retain one stable +logical identity. The output directory is ``graphify-out`` by default and overridable with the -``GRAPHIFY_OUT`` env var (worktrees or shared-output setups, #686). It accepts a -relative name (``"graphify-out-feature"``) or an absolute path +``GRAPHIFY_OUT`` environment variable (worktrees or shared-output setups, #686). +It accepts a relative name (``"graphify-out-feature"``) or an absolute path (``"/shared/graphify-out"``). This used to be duplicated as an identical ``_GRAPHIFY_OUT`` constant in ``__main__``, ``cache``, and ``watch``, while ``security`` and ``callflow_html`` hardcoded the literal ``"graphify-out"`` and silently ignored the override (#1423). Centralising it here keeps the name in one place. The value is read -once at import time, matching the previous per-module constants — set -``GRAPHIFY_OUT`` before the process starts (the normal worktree/shared-output -flow) and every reader honours it. +once at import time, matching the previous per-module constants; set +``GRAPHIFY_OUT`` before the process starts and every reader honours it. """ from __future__ import annotations import json +import ntpath import os import re import stat +import sys import tempfile +from collections.abc import Callable, Iterator from pathlib import Path, PurePosixPath +from typing import Any GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +PathLike = str | os.PathLike[str] + + +def io_path(path: PathLike) -> str: + r"""Return a path string suitable for Windows file-system I/O. + + Windows' legacy Win32 namespace rejects paths near ``MAX_PATH`` unless the + machine-wide long-path policy is enabled. The extended-length path syntax + does not depend on that policy: local paths use ``\\?\C:\...`` + and UNC paths use ``\\?\UNC\server\share\...``. + + Keep this conversion at the I/O boundary only. Graph IDs, manifests, + diagnostics, ignore matching, and user-visible paths should retain their + ordinary spelling, because the extended prefix is an API transport detail. + """ + value = os.fspath(path) + if sys.platform != "win32": + return value + if value.startswith(("\\\\?\\", "\\\\.\\")): + return value + + # Extended-length paths must be absolute and use backslashes. ntpath is + # used explicitly so this helper is unit-testable from non-Windows CI. + value = ntpath.abspath(value) + if value.startswith("\\\\"): + return "\\\\?\\UNC\\" + value[2:] + return "\\\\?\\" + value + + +def logical_path(path: PathLike) -> str: + r"""Remove a Windows extended-length prefix from a path string. + + ``os.walk(io_path(root))`` yields prefixed directory names. Strip the + transport prefix before paths enter graphify's matching/storage logic so a + long path does not acquire a second identity merely because it was opened + through the Windows extended namespace. + """ + value = os.fspath(path) + if value.upper().startswith("\\\\?\\UNC\\"): + return "\\\\" + value[8:] + if value.startswith("\\\\?\\"): + return value[4:] + return value + + +def resolve_path(path: PathLike) -> Path: + """Resolve ``path`` through the I/O namespace and return ordinary spelling. + + This mirrors ``Path.resolve(strict=False)`` while ensuring the operation can + reach a long Windows path even when the host's global long-path policy is + disabled. The returned ``Path`` intentionally has no extended prefix. + """ + return Path(logical_path(os.path.realpath(io_path(path)))) + + +def path_exists(path: PathLike) -> bool: + """Return whether ``path`` exists, using Windows-safe path spelling.""" + return os.path.exists(io_path(path)) + + +def path_is_file(path: PathLike) -> bool: + """Return whether ``path`` is a regular file, using Windows-safe spelling.""" + return os.path.isfile(io_path(path)) + + +def path_is_dir(path: PathLike) -> bool: + """Return whether ``path`` is a directory, using Windows-safe spelling.""" + return os.path.isdir(io_path(path)) + + +def path_is_symlink(path: PathLike) -> bool: + """Return whether ``path`` is a symbolic link, using Windows-safe spelling.""" + return os.path.islink(io_path(path)) + + +def path_stat(path: PathLike, *, follow_symlinks: bool = True) -> os.stat_result: + """Stat ``path`` through the Windows-safe I/O namespace.""" + return os.stat(io_path(path), follow_symlinks=follow_symlinks) + + +def make_dirs( + path: PathLike, + mode: int = 0o777, + *, + exist_ok: bool = False, +) -> None: + """Create a directory tree through the Windows-safe I/O namespace.""" + os.makedirs(io_path(path), mode=mode, exist_ok=exist_ok) + + +def scandir_path(path: PathLike) -> os.ScandirIterator[str]: + """Return ``os.scandir`` for ``path`` using Windows-safe spelling.""" + return os.scandir(io_path(path)) + + +def iterdir_path(path: PathLike) -> Iterator[Path]: + r"""Yield direct children while preserving the caller's path spelling. + + ``DirEntry.path`` inherits the extended absolute root supplied to + :func:`os.scandir`. Rebuild each child from the logical input root and the + entry name so a relative input remains relative and ``\\?\`` never escapes. + """ + logical_root = Path(logical_path(path)) + with scandir_path(path) as entries: + for entry in entries: + yield logical_root / entry.name + + +def glob_paths(path: PathLike, pattern: str) -> Iterator[Path]: + """Yield glob matches below ``path`` while retaining logical spelling. + + Prefix the root before delegating to :meth:`Path.glob`; this preserves + pathlib's matching behavior (including dotfiles) while every recursive + ``scandir`` stays in the extended Windows namespace. Matches are then + reconstructed relative to the caller's root, avoiding an accidental + relative-to-absolute API change on Windows. + """ + logical_root = Path(logical_path(path)) + filesystem_root = Path(io_path(path)) + for match in filesystem_root.glob(pattern): + try: + relative = match.relative_to(filesystem_root) + except ValueError: + # Defensive fallback for an unusual pathlib implementation; glob + # matches should ordinarily remain below their root. + yield Path(logical_path(match)) + else: + yield logical_root / relative + + +def unlink_path(path: PathLike, *, missing_ok: bool = False) -> None: + """Remove a file or symlink through the Windows-safe namespace.""" + try: + os.unlink(io_path(path)) + except FileNotFoundError: + if not missing_ok: + raise + + +def replace_path(source: PathLike, destination: PathLike) -> None: + """Atomically replace ``destination`` using Windows-safe path spellings.""" + os.replace(io_path(source), io_path(destination)) + + +def walk_path( + path: PathLike, + *, + topdown: bool = True, + onerror: Callable[[OSError], Any] | None = None, + followlinks: bool = False, +) -> Iterator[tuple[str, list[str], list[str]]]: + """Walk ``path`` safely and yield ordinary, user-facing path spellings. + + ``os.walk`` must receive the extended Windows form at the traversal root; + otherwise a later ``scandir`` fails as soon as a descendant crosses the + legacy ``MAX_PATH`` boundary. The extended prefix is stripped from every + yielded directory and from errors before control returns to callers. A + relative input remains relative even though the Windows I/O root must be + absolute before it can use the extended namespace. + """ + logical_root = logical_path(path) + filesystem_root = io_path(path) + ordinary_filesystem_root = logical_path(filesystem_root) + + def _from_filesystem_path(value: PathLike) -> str: + ordinary = logical_path(value) + if sys.platform != "win32": + return ordinary + try: + relative = ntpath.relpath(ordinary, ordinary_filesystem_root) + except ValueError: + # Different UNC shares/drives cannot be relativized; stripping the + # transport prefix is still the safest public representation. + return ordinary + if relative == ".": + return logical_root + return ntpath.join(logical_root, relative) + + def _onerror(error: OSError) -> None: + filename = getattr(error, "filename", None) + if filename is not None: + try: + error.filename = _from_filesystem_path(filename) + except (AttributeError, TypeError): + pass + if onerror is not None: + onerror(error) + + handler = _onerror if onerror is not None else None + for dirpath, dirnames, filenames in os.walk( + filesystem_root, + topdown=topdown, + onerror=handler, + followlinks=followlinks, + ): + yield _from_filesystem_path(dirpath), dirnames, filenames + + +def read_bytes(path: PathLike, *, limit: int | None = None) -> bytes: + """Read bytes through the Windows-safe I/O spelling of ``path``. + + ``limit`` mirrors ``file.read(limit)`` and supports bounded probes without a + separate, long-path-unsafe ``Path.open`` call. + """ + with open(io_path(path), "rb") as fh: + return fh.read() if limit is None else fh.read(limit) + + +def read_text( + path: PathLike, + *, + encoding: str | None = None, + errors: str | None = None, +) -> str: + """Read text through the Windows-safe I/O spelling of ``path``. + + The default encoding intentionally mirrors :meth:`Path.read_text` and the + built-in :func:`open`; corpus readers that require UTF-8 pass it explicitly. + """ + with open(io_path(path), "r", encoding=encoding, errors=errors) as fh: + return fh.read() + + +def write_text( + path: PathLike, + text: str, + *, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, +) -> int: + """Write text through the Windows-safe I/O spelling of ``path``.""" + with open( + io_path(path), + "w", + encoding=encoding, + errors=errors, + newline=newline, + ) as fh: + return fh.write(text) + + +def write_bytes(path: PathLike, data: bytes) -> int: + """Write bytes through the Windows-safe I/O spelling of ``path``.""" + with open(io_path(path), "wb") as fh: + return fh.write(data) + + def _atomic_replace(path: "str | Path", write_fn) -> None: """Atomically replace ``path`` with content written by ``write_fn(f)``. @@ -43,9 +302,13 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: """ # Resolve symlinks so the temp lands on the target's filesystem (same-fs # atomic rename) and the replace writes through the link, not over it. - real = Path(os.path.realpath(str(path))) - real.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = tempfile.mkstemp(dir=str(real.parent), prefix=f".{real.name}.", suffix=".tmp") + real = resolve_path(path) + make_dirs(real.parent, exist_ok=True) + fd, tmp = tempfile.mkstemp( + dir=io_path(real.parent), + prefix=f".{real.name}.", + suffix=".tmp", + ) try: with os.fdopen(fd, "w", encoding="utf-8") as f: write_fn(f) @@ -54,7 +317,7 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: # silently tightens a previously group/world-readable output to # owner-only. Best-effort — a chmod failure must not fail the write. try: - mode = stat.S_IMODE(os.stat(real).st_mode) + mode = stat.S_IMODE(path_stat(real).st_mode) except OSError: umask = os.umask(0) os.umask(umask) @@ -64,13 +327,13 @@ def _atomic_replace(path: "str | Path", write_fn) -> None: except OSError: pass try: - os.replace(tmp, str(real)) + os.replace(tmp, io_path(real)) except PermissionError: # Windows: os.replace fails (WinError 5/32) when the destination is # briefly locked by another handle (antivirus, an open reader). Fall # back to copy-then-delete, matching graphify.cache's atomic writer. import shutil - shutil.copy2(tmp, str(real)) + shutil.copy2(tmp, io_path(real)) os.unlink(tmp) except BaseException: try: @@ -335,7 +598,7 @@ def load_node_link_graph(path_or_data): p = Path(data) from graphify.security import check_graph_file_size_cap # lazy: security imports paths check_graph_file_size_cap(p) - data = json.loads(p.read_text(encoding="utf-8")) + data = json.loads(read_text(p, encoding="utf-8")) if isinstance(data, dict) and "links" not in data and "edges" in data: data = dict(data, links=data["edges"]) try: diff --git a/graphify/security.py b/graphify/security.py index 2dbe5bd77..d3dbebabf 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -15,7 +15,13 @@ import ipaddress import socket -from graphify.paths import GRAPHIFY_OUT, GRAPHIFY_OUT_NAME +from graphify.paths import ( + GRAPHIFY_OUT, + GRAPHIFY_OUT_NAME, + path_exists, + path_stat, + resolve_path, +) _ALLOWED_SCHEMES = {"http", "https"} _MAX_FETCH_BYTES = 52_428_800 # 50 MB hard cap for binary downloads @@ -324,22 +330,22 @@ def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: FileNotFoundError - resolved path does not exist """ if base is None: - resolved_hint = Path(path).resolve() + resolved_hint = resolve_path(path) for candidate in [resolved_hint, *resolved_hint.parents]: if candidate.name == GRAPHIFY_OUT_NAME: base = candidate break if base is None: - base = Path(GRAPHIFY_OUT).resolve() + base = resolve_path(GRAPHIFY_OUT) - base = base.resolve() - if not base.exists(): + base = resolve_path(base) + if not path_exists(base): raise ValueError( f"Graph base directory does not exist: {base}. " "Run /graphify first to build the graph." ) - resolved = Path(path).resolve() + resolved = resolve_path(path) try: resolved.relative_to(base) except ValueError: @@ -348,7 +354,7 @@ def validate_graph_path(path: str | Path, base: Path | None = None) -> Path: "Only paths inside graphify-out/ are permitted." ) - if not resolved.exists(): + if not path_exists(resolved): raise FileNotFoundError(f"Graph file not found: {resolved}") return resolved @@ -372,7 +378,7 @@ def check_graph_file_size_cap(path: Path) -> None: """ cap = _max_graph_file_bytes() try: - size = path.stat().st_size + size = path_stat(path).st_size except OSError: return if size > cap: diff --git a/graphify/symbol_resolution.py b/graphify/symbol_resolution.py index 892f31065..d75c4717c 100644 --- a/graphify/symbol_resolution.py +++ b/graphify/symbol_resolution.py @@ -5,11 +5,16 @@ import ast import re import unicodedata +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from collections.abc import Sequence from typing import Any +from graphify.paths import ( + read_text as _read_file_text, + resolve_path as _resolve_path, +) + from graphify.ids import make_id as _shared_make_id from graphify.paths import disambiguate_ambiguous_candidates from graphify.security import sanitize_metadata @@ -135,7 +140,7 @@ def parse_python_import_aliases(path: Path) -> dict[str, ImportedSymbol]: """ try: - source = path.read_text(encoding="utf-8", errors="replace") + source = _read_file_text(path, encoding="utf-8", errors="replace") tree = ast.parse(source) except (OSError, SyntaxError): return {} @@ -395,7 +400,7 @@ def _file_node_id_for_path(path: Path, root: Path) -> str: # node instead of an orphan. _bash_make_id / _bash_file_stem are exact copies # of extract._make_id / extract._file_stem, so IDs match. try: - rel = path.resolve().relative_to(root.resolve()) + rel = _resolve_path(path).relative_to(_resolve_path(root)) except ValueError: return _bash_make_id(str(path)) # path outside root: hash absolute path as fallback return _bash_make_id(_bash_file_stem(rel)) @@ -427,7 +432,7 @@ def resolve_bash_source_edges( - Inputs of type ``str`` and ``pathlib.Path`` are processed. Anything else is silently skipped. """ - path_by_index = [Path(p).resolve() for p in paths] + path_by_index = [_resolve_path(p) for p in paths] file_nid_by_path = {p: _file_node_id_for_path(p, root) for p in path_by_index} # resolved paths only functions_by_file: dict[str, dict[str, str]] = {} @@ -478,7 +483,7 @@ def resolve_bash_source_edges( if not candidate.is_absolute(): candidate = path.parent / candidate try: - target_path = candidate.resolve() + target_path = _resolve_path(candidate) except (OSError, RuntimeError): continue target_file_nid = file_nid_by_path.get(target_path) diff --git a/graphify/transcribe.py b/graphify/transcribe.py index 8e45bcaa2..6f23022e5 100644 --- a/graphify/transcribe.py +++ b/graphify/transcribe.py @@ -5,7 +5,14 @@ import os from pathlib import Path -from graphify.paths import out_path as _out_path +from graphify.paths import ( + glob_paths as _glob_paths, + io_path as _io_path, + make_dirs as _make_dirs, + out_path as _out_path, + path_exists as _path_exists, + write_text as _write_file_text, +) VIDEO_EXTENSIONS = {'.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.mp3', '.wav', '.m4a', '.ogg'} @@ -56,7 +63,7 @@ def download_audio(url: str, output_dir: Path) -> Path: from graphify.security import validate_url validate_url(url) # blocks private IPs, bad schemes before yt-dlp runs yt_dlp = _get_yt_dlp() - output_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(output_dir, exist_ok=True) # yt-dlp uses %(title)s which can be long/weird — use a stable name based on URL hash import hashlib @@ -66,7 +73,7 @@ def download_audio(url: str, output_dir: Path) -> Path: # Check for already-downloaded file for ext in ('.m4a', '.opus', '.mp3', '.ogg', '.wav', '.webm'): candidate = output_dir / f"yt_{url_hash}{ext}" - if candidate.exists(): + if _path_exists(candidate): print(f" cached audio: {candidate.name}") return candidate @@ -84,9 +91,9 @@ def download_audio(url: str, output_dir: Path) -> Path: info = ydl.extract_info(url, download=True) ext = info.get('ext', 'm4a') downloaded = output_dir / f"yt_{url_hash}.{ext}" - if not downloaded.exists(): + if not _path_exists(downloaded): # yt-dlp may have picked a different extension - for p in output_dir.glob(f"yt_{url_hash}.*"): + for p in _glob_paths(output_dir, f"yt_{url_hash}.*"): downloaded = p break return downloaded @@ -131,7 +138,7 @@ def transcribe( force: re-transcribe even if transcript already exists. """ out_dir = Path(output_dir) if output_dir else Path(_TRANSCRIPTS_DIR) - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) if is_url(str(video_path)): audio_path = download_audio(str(video_path), out_dir / "downloads") @@ -139,7 +146,7 @@ def transcribe( audio_path = Path(video_path) transcript_path = out_dir / (audio_path.stem + ".txt") - if transcript_path.exists() and not force: + if _path_exists(transcript_path) and not force: return transcript_path WhisperModel = _get_whisper() @@ -149,7 +156,7 @@ def transcribe( print(f" transcribing {audio_path.name} (model={model_name}) ...", flush=True) model = WhisperModel(model_name, device="cpu", compute_type="int8") segments, info = model.transcribe( - str(audio_path), + _io_path(audio_path), beam_size=5, initial_prompt=prompt, ) @@ -157,7 +164,7 @@ def transcribe( lines = [segment.text.strip() for segment in segments if segment.text.strip()] transcript = "\n".join(lines) - transcript_path.write_text(transcript, encoding="utf-8") + _write_file_text(transcript_path, transcript, encoding="utf-8") lang = info.language if hasattr(info, "language") else "unknown" print(f" transcript saved -> {transcript_path} (lang={lang}, {len(lines)} segments)") return transcript_path diff --git a/graphify/watch.py b/graphify/watch.py index a1adb64bb..5531ec571 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -10,7 +10,20 @@ from pathlib import Path # Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. -from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import ( + GRAPHIFY_OUT as _GRAPHIFY_OUT, + glob_paths as _glob_paths, + io_path as _io_path, + make_dirs as _make_dirs, + path_exists as _path_exists, + path_is_dir as _path_is_dir, + path_is_file as _path_is_file, + read_text as _read_text, + replace_path as _replace_path, + resolve_path as _resolve_path, + unlink_path as _unlink_path, + write_text as _write_text, +) _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 @@ -30,10 +43,10 @@ def _queue_pending(out_dir: Path, changed_paths: list[Path]) -> None: """ if not changed_paths: return - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) pending = out_dir / _PENDING_FILENAME payload = "".join(f"{os.fspath(p)}\n" for p in changed_paths) - with open(pending, "a", encoding="utf-8") as fh: + with open(_io_path(pending), "a", encoding="utf-8") as fh: fh.write(payload) @@ -45,10 +58,10 @@ def _drain_pending(out_dir: Path) -> list[Path]: fragment cannot poison the merge. """ pending = out_dir / _PENDING_FILENAME - if not pending.exists(): + if not _path_exists(pending): return [] try: - raw = pending.read_text(encoding="utf-8") + raw = _read_text(pending, encoding="utf-8") except OSError: return [] # Unlink BEFORE returning so a crash between read and process retains the @@ -57,7 +70,7 @@ def _drain_pending(out_dir: Path) -> list[Path]: # bug. Use missing_ok to tolerate a racing drain on platforms where # rename/unlink may interleave. with contextlib.suppress(FileNotFoundError): - pending.unlink() + _unlink_path(pending) seen: set[str] = set() out: list[Path] = [] for line in raw.splitlines(): @@ -89,10 +102,10 @@ def _write_build_config( if not excludes and gitignore is None: return try: - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) path = out_dir / _BUILD_CONFIG_FILENAME try: - config = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + config = json.loads(_read_text(path, encoding="utf-8")) if _path_is_file(path) else {} except (OSError, json.JSONDecodeError): config = {} if not isinstance(config, dict): @@ -101,7 +114,7 @@ def _write_build_config( config["excludes"] = list(excludes) if gitignore is not None: config["gitignore"] = gitignore - path.write_text(json.dumps(config), encoding="utf-8") + _write_text(path, json.dumps(config), encoding="utf-8") except OSError: pass @@ -110,8 +123,8 @@ def _read_build_excludes(out_dir: Path) -> list[str]: """Return the persisted ``--exclude`` patterns for this graph, or [].""" try: path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) + if _path_is_file(path): + cfg = json.loads(_read_text(path, encoding="utf-8")) ex = cfg.get("excludes") if isinstance(cfg, dict) else None if isinstance(ex, list): return [str(x) for x in ex if isinstance(x, str) and x] @@ -124,8 +137,8 @@ def _read_build_gitignore(out_dir: Path) -> bool: """Return whether rebuilds should honor VCS ignore files (default True).""" try: path = out_dir / _BUILD_CONFIG_FILENAME - if path.is_file(): - cfg = json.loads(path.read_text(encoding="utf-8")) + if _path_is_file(path): + cfg = json.loads(_read_text(path, encoding="utf-8")) if isinstance(cfg, dict) and isinstance(cfg.get("gitignore"), bool): return cfg["gitignore"] except (OSError, json.JSONDecodeError): @@ -175,12 +188,12 @@ def _rebuild_lock(out_dir: Path, *, blocking: bool = False): yield True return - out_dir.mkdir(parents=True, exist_ok=True) + _make_dirs(out_dir, exist_ok=True) lock_path = out_dir / ".rebuild.lock" # "a+" creates the file if missing without truncating an existing holder's # PID payload — important because another process may have already written # its PID before we attempt the flock. - fh = open(lock_path, "a+", encoding="utf-8") + fh = open(_io_path(lock_path), "a+", encoding="utf-8") acquired = False try: flags = fcntl.LOCK_EX if blocking else (fcntl.LOCK_EX | fcntl.LOCK_NB) @@ -211,7 +224,7 @@ def _rebuild_lock(out_dir: Path, *, blocking: bool = False): # unlinks; a non-acquiring caller leaves the existing lock in place. if acquired: with contextlib.suppress(OSError): - lock_path.unlink() + _unlink_path(lock_path) def _apply_resource_limits() -> None: @@ -298,14 +311,14 @@ def _changed_path_candidates(raw: Path, *, change_root: Path, watch_root: Path) """ if raw.is_absolute(): lexical = Path(os.path.abspath(raw)) - resolved = raw.resolve() + resolved = _resolve_path(raw) return [lexical] if lexical == resolved else [lexical, resolved] candidates: list[Path] = [] seen: set[str] = set() for base in (change_root, watch_root): lexical = Path(os.path.abspath(base / raw)) - for cand in (lexical, lexical.resolve()): + for cand in (lexical, _resolve_path(lexical)): key = os.fspath(cand) if key in seen: continue @@ -324,7 +337,7 @@ def _relativize_source_files(payload: dict, root: Path, *, scope: Path | None = if not source_path.is_absolute(): continue try: - resolved = source_path.resolve() + resolved = _resolve_path(source_path) if scope is not None and not _is_relative_to(resolved, scope): continue item["source_file"] = resolved.relative_to(root).as_posix() @@ -366,14 +379,14 @@ def __init__( relative_marker_prefix: str | None = None root_marker = out / ".graphify_root" - if root_marker.exists(): + if _path_exists(root_marker): try: - saved_root = Path(root_marker.read_text(encoding="utf-8").strip()) + saved_root = Path(_read_text(root_marker, encoding="utf-8").strip()) if saved_root.is_absolute(): - self.existing_source_root = saved_root.resolve() + self.existing_source_root = _resolve_path(saved_root) else: - invocation_root = Path.cwd().resolve() - if (invocation_root / saved_root).resolve() == watch_root: + invocation_root = _resolve_path(Path.cwd()) + if _resolve_path(invocation_root / saved_root) == watch_root: self.existing_source_root = invocation_root relative_marker_prefix = posixpath.normpath(saved_root.as_posix()) except (OSError, ValueError): @@ -470,7 +483,7 @@ def _reconcile_existing_graph( ) -> tuple[dict, dict]: """Merge fresh extraction with preserved graph entries and evict stale sources.""" existing_graph_data: dict = {} - if not existing_graph.exists(): + if not _path_exists(existing_graph): return result, existing_graph_data # Fail-closed load (#2251): reuse build._load_existing_graph, which raises @@ -488,7 +501,7 @@ def _reconcile_existing_graph( # topology compare) the (nodes, edges, hyperedges) tuple does not carry. # A failure here (e.g. a race rewriting the file) still propagates, # staying fail-closed. - existing = json.loads(existing_graph.read_text(encoding="utf-8")) + existing = json.loads(_read_text(existing_graph, encoding="utf-8")) existing_graph_data = existing # Backfill tier provenance on legacy items (#2334), mirroring @@ -559,7 +572,7 @@ def _reconcile_existing_graph( if identity: alive = _alive_cache.get(identity) if alive is None: - alive = Path(identity).exists() + alive = _path_exists(identity) _alive_cache[identity] = alive if not alive: normalized = source_paths.normalize(source_file) @@ -573,7 +586,7 @@ def _reconcile_existing_graph( if identity: alive = _alive_cache.get(identity) if alive is None: - alive = Path(identity).exists() + alive = _path_exists(identity) _alive_cache[identity] = alive if alive: excluded_alive_files.add(identity) @@ -840,7 +853,7 @@ def _accounted(n: dict) -> bool: if all(_accounted(n) for n in lost): return True if tmp is not None: - tmp.unlink(missing_ok=True) + _unlink_path(tmp, missing_ok=True) print( f"[graphify] WARNING: new graph has {len(new_nodes)} nodes but existing " f"graph.json has {len(existing_nodes)}. Refusing to overwrite — you may be " @@ -872,7 +885,7 @@ def _stabilize_rebuild_cwd(watch_path: Path) -> bool: return True repo_root = os.environ.get("GRAPHIFY_REPO_ROOT", "").strip() - if repo_root and Path(repo_root).is_dir(): + if repo_root and _path_is_dir(repo_root): try: os.chdir(repo_root) return True @@ -939,7 +952,7 @@ def _rebuild_code( with _rebuild_lock(out, blocking=block_on_lock) as got: if not got: print("[graphify watch] Rebuild already in progress for " - f"{watch_path.resolve()} - changes queued.") + f"{_resolve_path(watch_path)} - changes queued.") return False # Lock acquired. Drain anything queued by earlier contenders # (including, importantly, the paths we just queued ourselves) @@ -978,12 +991,12 @@ def _rebuild_code( ) and ok return ok - watch_root = watch_path.resolve() + watch_root = _resolve_path(watch_path) # project_root stays CWD-anchored for a relative invocation on purpose: the # persisted graph rehomes source_file across invocation styles against it # (tests/test_watch.py:1389, :1428). The manifest is a different artifact # with a different anchor — see the save_manifest calls below. - project_root = Path.cwd().resolve() if not watch_path.is_absolute() else watch_root + project_root = _resolve_path(Path.cwd()) if not watch_path.is_absolute() else watch_root report_root = _report_root_label(watch_path) try: from graphify.extract import extract, _get_extractor @@ -1015,7 +1028,7 @@ def _rebuild_code( ast_doc_files.append(p) existing_graph = out / "graph.json" - if not code_files and not existing_graph.exists(): + if not code_files and not _path_exists(existing_graph): print("[graphify watch] No code files found - nothing to rebuild.") return False @@ -1034,10 +1047,10 @@ def _rebuild_code( # graph must be allowed to self-heal on a full rebuild without the # shrink-guard refusing the smaller write. semantic_doc_files: set[Path] = set() - if ast_doc_files and existing_graph.exists(): + if ast_doc_files and _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - prior = json.loads(existing_graph.read_text(encoding="utf-8")) + prior = json.loads(_read_text(existing_graph, encoding="utf-8")) prior_paths = _StoredSourcePaths( prior, out=out, @@ -1102,14 +1115,17 @@ def _add_deleted_source(path: Path) -> None: # an incremental rebuild, or their semantic nodes would be wiped. semantic_doc_set = {Path(os.path.abspath(p)) for p in semantic_doc_files} wanted: list[Path] = [] - change_root = Path.cwd().resolve() + change_root = _resolve_path(Path.cwd()) for raw in changed_paths: candidates = _changed_path_candidates( raw, change_root=change_root, watch_root=watch_root, ) - tracked = next((cand for cand in candidates if cand.exists() and cand in code_set), None) + tracked = next( + (cand for cand in candidates if _path_exists(cand) and cand in code_set), + None, + ) if tracked is not None: if tracked not in wanted and tracked not in semantic_doc_set: wanted.append(tracked) @@ -1118,7 +1134,7 @@ def _add_deleted_source(path: Path) -> None: existing_in_root = next( ( cand for cand in candidates - if cand.exists() and _is_relative_to(cand, watch_root) + if _path_exists(cand) and _is_relative_to(cand, watch_root) ), None, ) @@ -1209,7 +1225,7 @@ def _add_deleted_source(path: Path) -> None: else: rebuilt_sources = {(_nsf(str(p), _rebuilt_root) or str(p)) for p in extract_targets} rebuilt_sources |= set(deleted_paths) - out.mkdir(exist_ok=True) + _make_dirs(out, exist_ok=True) if no_cluster: # Normalise to "links" key so schema is consistent with the full clustered path. @@ -1228,10 +1244,10 @@ def _add_deleted_source(path: Path) -> None: } candidate_graph_text = _json_text(candidate_graph_data) same_graph = False - if existing_graph.exists(): + if _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) + existing_payload = json.loads(_read_text(existing_graph, encoding="utf-8")) except Exception as exc: # A load failure is NOT "graph changed" (#2251): refuse to # overwrite a graph we merely failed to read. Normally @@ -1263,12 +1279,12 @@ def _add_deleted_source(path: Path) -> None: # Atomic replace via tmp file, matching the clustered path: a # crash mid-write must not leave a truncated graph.json. graph_tmp = out / ".graph.tmp.json" - graph_tmp.write_text(candidate_graph_text, encoding="utf-8") - graph_tmp.replace(existing_graph) + _write_text(graph_tmp, candidate_graph_text, encoding="utf-8") + _replace_path(graph_tmp, existing_graph) # Write the user-supplied path only after the candidate graph is # accepted, so a refused shrink cannot mismatch graph and marker. - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + _write_text(out / ".graphify_root", str(watch_path), encoding="utf-8") try: from graphify.detect import save_manifest @@ -1286,8 +1302,8 @@ def _add_deleted_source(path: Path) -> None: # clear stale needs_update flag if present flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) if same_graph: print("[graphify watch] No code-graph changes detected (--no-cluster); outputs left untouched.") @@ -1331,8 +1347,8 @@ def _add_deleted_source(path: Path) -> None: except Exception: pass flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) print("[graphify watch] No code-graph topology changes detected; outputs left untouched.") return True @@ -1346,7 +1362,11 @@ def _add_deleted_source(path: Path) -> None: labels_file = out / ".graphify_labels.json" sig_file = out / (".graphify_labels.json" + ".sig") try: - raw = json.loads(labels_file.read_text(encoding="utf-8")) if labels_file.exists() else {} + raw = ( + json.loads(_read_text(labels_file, encoding="utf-8")) + if _path_exists(labels_file) + else {} + ) # Skip persisted "Community N" placeholders so the hub-fill below # replaces them instead of perpetuating them on every rebuild (#2073). labels = { @@ -1367,11 +1387,11 @@ def _add_deleted_source(path: Path) -> None: from graphify.cluster import community_member_sigs cur_sigs = community_member_sigs(communities) saved_sigs: dict[int, str] = {} - if sig_file.exists(): + if _path_exists(sig_file): try: saved_sigs = { int(k): v for k, v in - json.loads(sig_file.read_text(encoding="utf-8")).items() + json.loads(_read_text(sig_file, encoding="utf-8")).items() if isinstance(v, str) } except Exception: @@ -1411,19 +1431,19 @@ def _add_deleted_source(path: Path) -> None: json_written = to_json(G, communities, str(graph_tmp), force=True, built_at_commit=commit, community_labels=labels) if not json_written: return False - candidate_graph_data = json.loads(graph_tmp.read_text(encoding="utf-8")) + candidate_graph_data = json.loads(_read_text(graph_tmp, encoding="utf-8")) same_graph = False same_report = False - if existing_graph.exists(): + if _path_exists(existing_graph): try: check_graph_file_size_cap(existing_graph) - existing_payload = json.loads(existing_graph.read_text(encoding="utf-8")) + existing_payload = json.loads(_read_text(existing_graph, encoding="utf-8")) except Exception as exc: # A load failure is NOT "graph changed" (#2251): refuse to # overwrite a graph we merely failed to read. Normally # unreachable — the reconcile load above already failed # closed — but a race rewriting the file can land here. - graph_tmp.unlink(missing_ok=True) + _unlink_path(graph_tmp, missing_ok=True) print( f"error: Cannot read {existing_graph}: {exc}. " "Refusing to overwrite; delete the file and run a " @@ -1438,12 +1458,12 @@ def _add_deleted_source(path: Path) -> None: ) except Exception: same_graph = False - if report_path.exists(): - old_report = report_path.read_text(encoding="utf-8") + if _path_exists(report_path): + old_report = _read_text(report_path, encoding="utf-8") same_report = _report_for_compare(old_report) == _report_for_compare(report) no_change = same_graph and same_report if no_change: - graph_tmp.unlink(missing_ok=True) + _unlink_path(graph_tmp, missing_ok=True) print("[graphify watch] No code-graph changes detected; graph.json/GRAPH_REPORT.md left untouched.") else: if not _check_shrink( @@ -1455,17 +1475,20 @@ def _add_deleted_source(path: Path) -> None: return False from graphify.export import backup_if_protected as _backup _backup(out) - graph_tmp.replace(existing_graph) - report_path.write_text(report, encoding="utf-8") - labels_file.write_text(labels_json, encoding="utf-8") + _replace_path(graph_tmp, existing_graph) + _write_text(report_path, report, encoding="utf-8") + _write_text(labels_file, labels_json, encoding="utf-8") # Keep the membership signatures in step with the labels we just wrote. # Skipping this was the other half of the stale-label bug: labels.json # advanced every rebuild while the sidecar kept describing an older # clustering, so the guard above had nothing accurate to check against. - sig_file.write_text( - json.dumps({str(k): v for k, v in cur_sigs.items()}), encoding="utf-8") + _write_text( + sig_file, + json.dumps({str(k): v for k, v in cur_sigs.items()}), + encoding="utf-8", + ) - (out / ".graphify_root").write_text(str(watch_path), encoding="utf-8") + _write_text(out / ".graphify_root", str(watch_path), encoding="utf-8") try: from graphify.detect import save_manifest @@ -1495,8 +1518,8 @@ def _add_deleted_source(path: Path) -> None: # the community-aggregation view in exactly this case, so do # the same here: current AND present beats current OR present. from graphify.exporters.html import _viz_node_limit - if html_target.exists(): - html_target.unlink() + if _path_exists(html_target): + _unlink_path(html_target) limit = _viz_node_limit() if limit <= 0: # GRAPHIFY_VIZ_NODE_LIMIT=0 means "no HTML viz" (CI runners), @@ -1508,14 +1531,14 @@ def _add_deleted_source(path: Path) -> None: community_labels=labels or None, node_limit=limit) # The aggregator declines to write a single-community # graph, so trust the file rather than the call. - html_written = html_target.exists() + html_written = _path_exists(html_target) except Exception as fallback_err: print(f"[graphify watch] Skipped graph.html: {viz_err} " f"(aggregated view also failed: {fallback_err})") # Regenerate callflow HTML if the user previously generated one — # opt-in by existence so users who never ran callflow-html aren't affected. - callflow_files = list(out.glob("*-callflow.html")) + callflow_files = list(_glob_paths(out, "*-callflow.html")) if callflow_files and not no_change: try: from graphify.callflow_html import write_callflow_html @@ -1532,8 +1555,8 @@ def _add_deleted_source(path: Path) -> None: # clear stale needs_update flag if present flag = out / "needs_update" - if flag.exists(): - flag.unlink() + if _path_exists(flag): + _unlink_path(flag) if not no_change: print(f"[graphify watch] Rebuilt: {G.number_of_nodes()} nodes, " @@ -1558,7 +1581,7 @@ def check_update(watch_path: Path) -> bool: that the update is needed. """ flag = Path(watch_path) / _GRAPHIFY_OUT / "needs_update" - if flag.exists(): + if _path_exists(flag): print(f"[graphify check-update] Pending non-code changes in {watch_path}.") print("[graphify check-update] Run `/graphify --update` to apply semantic re-extraction.") return True @@ -1567,8 +1590,8 @@ def check_update(watch_path: Path) -> bool: def _notify_only(watch_path: Path) -> None: """Write a flag file and print a notification (fallback for non-code-only corpora).""" flag = watch_path / _GRAPHIFY_OUT / "needs_update" - flag.parent.mkdir(parents=True, exist_ok=True) - flag.write_text("1", encoding="utf-8") + _make_dirs(flag.parent, exist_ok=True) + _write_text(flag, "1", encoding="utf-8") print(f"\n[graphify watch] New or changed files detected in {watch_path}") print("[graphify watch] Non-code files changed - semantic re-extraction requires LLM.") print("[graphify watch] Run `/graphify --update` in Claude Code to update the graph.") @@ -1607,7 +1630,7 @@ def watch(watch_path: Path, debounce: float = 3.0) -> None: # (Time Machine writes, Docker/Colima VM I/O, Spotlight indexing, …) — # without this short-circuit a busy volume can saturate a CPU core # discarding events one extension at a time. (gh-928) - watch_root_for_ignore = watch_path.resolve() + watch_root_for_ignore = _resolve_path(watch_path) ignore_patterns = _load_graphifyignore( watch_root_for_ignore, gitignore=_read_build_gitignore(watch_path / _GRAPHIFY_OUT), @@ -1646,7 +1669,7 @@ def on_any_event(self, event): observer.schedule(handler, str(watch_path), recursive=True) observer.start() - print(f"[graphify watch] Watching {watch_path.resolve()} - press Ctrl+C to stop") + print(f"[graphify watch] Watching {_resolve_path(watch_path)} - press Ctrl+C to stop") print(f"[graphify watch] Code changes rebuild graph automatically. " f"Doc/image changes require /graphify --update.") print(f"[graphify watch] Debounce: {debounce}s") diff --git a/tests/test_atomic_writes.py b/tests/test_atomic_writes.py index 968cb0b96..ef5986bc6 100644 --- a/tests/test_atomic_writes.py +++ b/tests/test_atomic_writes.py @@ -6,6 +6,7 @@ """ import json import os +import stat import pytest @@ -41,18 +42,26 @@ def test_write_text_atomic_preserves_existing_mode(tmp_path): p = tmp_path / "graph.json" p.write_text("{}", encoding="utf-8") os.chmod(p, 0o644) + # Windows exposes a synthetic permission mask (typically 0666 for a + # writable file) rather than preserving POSIX owner/group distinctions. + # Compare with the mode the platform actually reports after chmod instead + # of assuming that every OS can represent 0644 exactly. + expected_mode = stat.S_IMODE(os.stat(p).st_mode) write_text_atomic(p, '{"x": 1}') - assert (os.stat(p).st_mode & 0o777) == 0o644 + assert stat.S_IMODE(os.stat(p).st_mode) == expected_mode -def test_write_text_atomic_new_file_respects_umask(tmp_path): - # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's - # 0600 — otherwise every fresh graph.json would be owner-only. +def test_write_text_atomic_new_file_matches_platform_default_mode(tmp_path): + # A brand-new atomic file should have the same mode as an ordinary new file + # on this platform, NOT mkstemp's owner-only default. This also avoids + # assuming Windows can represent a POSIX umask-derived mode exactly. + reference = tmp_path / "reference.json" + reference.write_text("{}", encoding="utf-8") + expected_mode = stat.S_IMODE(os.stat(reference).st_mode) + p = tmp_path / "new.json" write_text_atomic(p, "{}") - umask = os.umask(0) - os.umask(umask) - assert (os.stat(p).st_mode & 0o777) == (0o666 & ~umask) + assert stat.S_IMODE(os.stat(p).st_mode) == expected_mode def test_write_text_atomic_writes_through_symlink(tmp_path): @@ -61,7 +70,15 @@ def test_write_text_atomic_writes_through_symlink(tmp_path): target = tmp_path / "real.json" target.write_text("old", encoding="utf-8") link = tmp_path / "link.json" - link.symlink_to(target) + try: + link.symlink_to(target) + except OSError as exc: + if os.name == "nt" and getattr(exc, "winerror", None) == 1314: + pytest.skip( + "Windows symlink creation requires Developer Mode, " + "administrator rights, or SeCreateSymbolicLinkPrivilege" + ) + raise write_text_atomic(link, "new") assert link.is_symlink() assert target.read_text() == "new" diff --git a/tests/test_cpp_preprocess.py b/tests/test_cpp_preprocess.py index 75ca8de5f..a6026576c 100644 --- a/tests/test_cpp_preprocess.py +++ b/tests/test_cpp_preprocess.py @@ -4,6 +4,8 @@ terminator, so _cpp_preprocess passes an absolute path which can never be parsed as a cpp option. """ +from pathlib import Path + from graphify import extract @@ -28,5 +30,8 @@ class _Result: out = extract._cpp_preprocess(f) assert out == b"preprocessed" last_arg = captured["argv"][-1] - assert last_arg.startswith("/"), f"path arg must be absolute, got {last_arg!r}" + assert Path(last_arg).is_absolute(), f"path arg must be absolute, got {last_arg!r}" assert not last_arg.startswith("-"), "path arg must never look like an option" + assert not last_arg.startswith("\\\\?\\"), ( + "transport-only Windows extended prefix must not cross into cpp" + ) diff --git a/tests/test_long_path_hashing.py b/tests/test_long_path_hashing.py index 8e4b3bd1a..f9fa13806 100644 --- a/tests/test_long_path_hashing.py +++ b/tests/test_long_path_hashing.py @@ -1,50 +1,49 @@ r"""#1655 — files whose absolute path exceeds Windows MAX_PATH (260) must still -be hashed, or their manifest entry never stabilizes and detect_incremental +be hashed, or their manifest entry never stabilizes and ``detect_incremental`` re-flags them as changed on every run. The plain file APIs reject long paths on win32 unless prefixed with the -extended-length marker `\\?\`. _os_path adds it (for I/O), the mirror of -cache._normalize_path which strips it (for stable keys). +extended-length marker ``\\?\``. :func:`graphify.paths.io_path` adds it for I/O, +while cache keys and manifests retain ordinary paths. """ from __future__ import annotations from pathlib import Path from graphify import detect +from graphify.paths import io_path def test_os_path_noop_on_posix(monkeypatch): monkeypatch.setattr("sys.platform", "linux") - p = Path("/home/user/deep/file.py") - assert detect._os_path(p) == str(p) + path = Path("/home/user/deep/file.py") + assert io_path(path) == str(path) def test_os_path_adds_prefix_on_win32(monkeypatch): monkeypatch.setattr("sys.platform", "win32") - # os.path.abspath is posix here, so exercise the already-qualified branch: - # a value that abspath leaves intact still gets the prefix. - out = detect._os_path(Path("/already/abs/file.py")) - assert out.startswith("\\\\?\\") + assert io_path(Path(r"C:\already\abs\file.py")) == r"\\?\C:\already\abs\file.py" def test_os_path_idempotent_on_win32(monkeypatch): monkeypatch.setattr("sys.platform", "win32") - already = "\\\\?\\C:\\a\\file.py" - assert detect._os_path(Path(already)) == already + already = r"\\?\C:\a\file.py" + assert io_path(Path(already)) == already def test_hashing_still_works_and_stabilizes(tmp_path): - # End-to-end (posix): a hashed file must produce a stable, non-empty hash so - # its manifest entry doesn't churn. Guards against the _os_path indirection - # breaking normal hashing. - f = tmp_path / "deep" / "nested" / "module.py" - f.parent.mkdir(parents=True) - f.write_text("def x():\n return 1\n") - h1 = detect._md5_file(f) - h2 = detect._md5_file(f) - assert h1 and h1 == h2 - - got = detect._stat_and_hash(str(f)) - assert got is not None - assert got[0] == str(f) - assert got[2] == h1 + # End-to-end (POSIX): a hashed file must produce a stable, non-empty hash so + # its manifest entry does not churn. This guards against the I/O adapter + # breaking ordinary Linux/macOS hashing while fixing Windows long paths. + source = tmp_path / "deep" / "nested" / "module.py" + source.parent.mkdir(parents=True) + source.write_text("def x():\n return 1\n", encoding="utf-8") + + first = detect._md5_file(source) + second = detect._md5_file(source) + assert first and first == second + + stat_and_hash = detect._stat_and_hash(str(source)) + assert stat_and_hash is not None + assert stat_and_hash[0] == str(source) + assert stat_and_hash[2] == first diff --git a/tests/test_windows_long_paths.py b/tests/test_windows_long_paths.py new file mode 100644 index 000000000..698716a42 --- /dev/null +++ b/tests/test_windows_long_paths.py @@ -0,0 +1,239 @@ +r"""Regression coverage for Windows extended-length path handling. + +Graphify keeps normal drive/UNC spellings as its public path identity and uses +``\\?\`` only when crossing an operating-system I/O boundary. These tests run +on every host; the deep-path integrations exercise native long paths on Windows +and verify that Linux/macOS remain unchanged. +""" +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from graphify.cache import file_hash +from graphify.detect import FileType, detect, load_manifest, save_manifest +from graphify.extract import collect_files, extract +from graphify.extractors.markdown import extract_markdown +from graphify.paths import ( + glob_paths, + io_path, + iterdir_path, + logical_path, + make_dirs, + path_exists, + path_is_file, + path_stat, + read_bytes, + read_text, + walk_path, + write_text, +) + + +def _deep_parent(root: Path, filename: str, *, minimum_length: int = 320) -> Path: + """Return a parent whose ordinary child path exceeds ``minimum_length``.""" + parent = root + index = 0 + while len(str(parent / filename)) <= minimum_length: + parent /= f"level_{index:02d}_{'x' * 28}" + index += 1 + return parent + + +def _remove_tree(path: Path) -> None: + """Clean up a deep Windows tree without relying on pytest's plain path I/O.""" + shutil.rmtree(io_path(path), ignore_errors=True) + + +def test_io_path_is_a_noop_off_windows(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "linux") + value = "/tmp/a/deep/file.py" + assert io_path(value) == value + assert logical_path(value) == value + + +@pytest.mark.parametrize( + ("ordinary", "extended"), + [ + (r"C:\repo\src\module.py", r"\\?\C:\repo\src\module.py"), + ( + r"\\server\share\manuals\deep\file.pdf", + r"\\?\UNC\server\share\manuals\deep\file.pdf", + ), + ], +) +def test_io_path_converts_windows_drive_and_unc_paths( + monkeypatch: pytest.MonkeyPatch, + ordinary: str, + extended: str, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + assert io_path(ordinary) == extended + assert logical_path(extended) == ordinary + assert io_path(extended) == extended + + +def test_io_path_normalizes_before_prefixing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "platform", "win32") + assert io_path(r"C:\repo\one\..\two/file.py") == r"\\?\C:\repo\two\file.py" + + +def test_walk_path_uses_extended_unc_and_yields_logical_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"\\server\share\root" + extended = r"\\?\UNC\server\share\root" + called_with: list[str] = [] + + def fake_walk(top, *, topdown, followlinks, onerror): + called_with.append(top) + assert topdown is True + assert followlinks is False + assert onerror is not None + yield top, ["deep"], ["root.py"] + yield top + r"\deep", [], ["nested.py"] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + rows = list(walk_path(root, onerror=lambda _error: None)) + + assert called_with == [extended] + assert rows == [ + (root, ["deep"], ["root.py"]), + (root + r"\deep", [], ["nested.py"]), + ] + + +def test_walk_path_preserves_relative_windows_spelling( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"relative\root" + + def fake_walk(top, *, topdown, followlinks, onerror): + del topdown, followlinks, onerror + yield top, ["deep"], ["root.py"] + yield top + r"\deep", [], ["nested.py"] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + rows = list(walk_path(root)) + + assert rows == [ + (root, ["deep"], ["root.py"]), + (root + r"\deep", [], ["nested.py"]), + ] + + +def test_walk_path_removes_prefix_from_reported_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + root = r"\\server\share\root" + reported: list[OSError] = [] + + def fake_walk(top, *, topdown, followlinks, onerror): + del topdown, followlinks + assert onerror is not None + onerror(FileNotFoundError(3, "not found", top + r"\too\deep")) + return [] + + monkeypatch.setattr("graphify.paths.os.walk", fake_walk) + assert list(walk_path(root, onerror=reported.append)) == [] + assert len(reported) == 1 + assert reported[0].filename == root + r"\too\deep" + + +def test_deep_path_discovery_hash_manifest_and_markdown_extraction(tmp_path: Path) -> None: + root = tmp_path / "corpus" + cache_root = tmp_path / "cache" + parent = _deep_parent(root, "notes.md") + source = parent / "notes.md" + manifest_path = parent / "graphify-out" / "manifest.json" + + try: + make_dirs(parent, exist_ok=True) + write_text(source, "# Deep manual\n\nLong-path content.\n", encoding="utf-8") + assert len(str(source)) > 300 + assert path_exists(source) + assert path_is_file(source) + assert path_stat(source).st_size > 0 + assert source in set(iterdir_path(parent)) + assert source in set(glob_paths(parent, "*.md")) + assert source in set(glob_paths(root, "**/*.md")) + assert read_bytes(source, limit=13) == b"# Deep manual" + + result = detect(root, cache_root=cache_root) + assert result["walk_errors"] == [] + assert str(source) in result["files"][FileType.DOCUMENT] + assert source in collect_files(root, root=root) + + first_hash = file_hash(source, root, cache_root=cache_root) + second_hash = file_hash(source, root, cache_root=cache_root) + assert first_hash and first_hash == second_hash + + save_manifest( + {FileType.DOCUMENT: [str(source)]}, + str(manifest_path), + root=root, + ) + loaded = load_manifest(str(manifest_path), root=root) + assert str(source) in loaded + assert "\\\\?\\" not in read_text(manifest_path, encoding="utf-8") + for cache_file in glob_paths(cache_root, "**/*.json"): + assert "\\\\?\\" not in read_text(cache_file, encoding="utf-8") + + extracted = extract_markdown(source) + assert any(node.get("label") == "Deep manual" for node in extracted["nodes"]) + + serialized = json.dumps({"detect": result, "extract": extracted}, default=str) + assert "\\\\?\\" not in serialized + finally: + _remove_tree(root) + + +def test_deep_python_path_full_extraction(tmp_path: Path) -> None: + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_python") + + root = tmp_path / "corpus" + cache_root = tmp_path / "cache" + parent = _deep_parent(root, "deep_module.py") + source = parent / "deep_module.py" + + try: + make_dirs(parent, exist_ok=True) + write_text( + source, + "class DeepThing:\n" + " def answer(self):\n" + " return 42\n", + encoding="utf-8", + ) + + first = extract( + [source], + root=root, + cache_root=cache_root, + parallel=False, + ) + # Exercise the warm-cache path as well as first-pass parsing. The 0.9.30 + # cache portability code resolves both the corpus root and source path; + # those lookups must cross the same Windows I/O boundary. + second = extract( + [source], + root=root, + cache_root=cache_root, + parallel=False, + ) + + for result in (first, second): + assert any(node.get("label") == "DeepThing" for node in result["nodes"]) + assert "\\\\?\\" not in json.dumps(result, default=str) + for cache_file in glob_paths(cache_root, "**/*.json"): + assert "\\\\?\\" not in read_text(cache_file, encoding="utf-8") + finally: + _remove_tree(root)