From a35415b1e1baafa9bc79afc741c0895f1b008426 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:02 -0400 Subject: [PATCH 1/8] feat(extract): add Common Lisp extractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an imperative tree-sitter-commonlisp walker that extracts packages, classes, functions, methods, macros, generic-function dispatch, and calls from Common Lisp source. Registered alongside the other custom extractors so homoiconic CL — where every defXXX form is the same list_lit node — is classified by reading definer symbols rather than tree-sitter node types. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- graphify/extract.py | 500 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 500 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index a40304b07..b44f74b10 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -10947,6 +10947,506 @@ def _body_of(block): return {"nodes": nodes, "edges": edges} +# ── Common Lisp ────────────────────────────────────────────────────────────── + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} + + +# ── Main extract and collect_files ──────────────────────────────────────────── + + +def _check_tree_sitter_version() -> None: + """Raise a clear error if tree-sitter is too old for the new Language API.""" + try: + from tree_sitter import LANGUAGE_VERSION + except ImportError: + raise ImportError( + "tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'" + ) + # Language API v2 starts at LANGUAGE_VERSION 14 + if LANGUAGE_VERSION < 14: + import tree_sitter as _ts + raise RuntimeError( + f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. " + f"graphify requires tree-sitter >= 0.23.0 (Language API v2). " + f"Run: pip install --upgrade tree-sitter" + ) + + + + _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, From da6f1fd58a47a5160908dc211f3f537523a2bb11 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:45 -0400 Subject: [PATCH 2/8] fix(extract): type Common Lisp rationale nodes so they don't leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emit docstring-derived rationale nodes with file_type "rationale" instead of "code" so the cross-file resolution filter excludes them, keeping docstrings out of import and symbol-resolution edges. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- graphify/extract.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index b44f74b10..d9d9354d1 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -11041,13 +11041,13 @@ def _cl_id(*parts: str) -> str: normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) return _make_id(*normalized) - def add_node(nid: str, label: str, line: int) -> None: + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: if nid not in seen_ids: seen_ids.add(nid) nodes.append({ "id": nid, "label": label, - "file_type": "code", + "file_type": file_type, "source_file": str_path, "source_location": f"L{line}", }) @@ -11214,7 +11214,7 @@ def _handle_defun_node(node) -> None: doc_text = _text(child).strip('"') if doc_text: doc_nid = _cl_id(func_nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") add_edge(doc_nid, func_nid, "rationale_for", child.start_point[0] + 1) break @@ -11304,7 +11304,7 @@ def _handle_def_form(node, def_keyword: str) -> None: doc_text = _text(child).strip('"') if doc_text: doc_nid = _cl_id(nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") add_edge(doc_nid, nid, "rationale_for", child.start_point[0] + 1) break From a0fc2f119f406828a21df8877ce8c29cc9e9260d Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:57 -0400 Subject: [PATCH 3/8] feat(detect): dispatch Common Lisp source files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Classify .lisp/.cl/.lsp/.asd as code and route them to the Common Lisp extractor so ASDF systems and source files build into the graph. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- graphify/detect.py | 2 +- graphify/extract.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/graphify/detect.py b/graphify/detect.py index 2eff3f80c..5cb8b3926 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -25,7 +25,7 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.lisp', '.cl', '.lsp', '.asd', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.csproj', '.fsproj', '.vbproj', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index d9d9354d1..edc6ca504 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -11500,6 +11500,10 @@ def _check_tree_sitter_version() -> None: ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".lisp": extract_commonlisp, + ".cl": extract_commonlisp, + ".lsp": extract_commonlisp, + ".asd": extract_commonlisp, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, From 60f959a4cb100c88c3512a9c0cefee6c5a4c144c Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:34:11 -0400 Subject: [PATCH 4/8] build: add tree-sitter-commonlisp as an optional extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Declare the Common Lisp grammar under optional-dependencies, matching the policy for other niche grammars, so the core install stays lean and Lisp users opt in with the commonlisp extra. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 85baa4843..fb11f47ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,12 +66,13 @@ gemini = ["openai", "tiktoken"] openai = ["openai", "tiktoken"] chinese = ["jieba"] sql = ["tree-sitter-sql"] +commonlisp = ["tree-sitter-commonlisp"] # tree-sitter-dm (BYOND DreamMaker) ships only a Windows wheel, so on Linux/Mac it # must compile from source (needs a C toolchain + python3-dev). Keeping it optional # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" From 23b4a4161a327739da80bb57e8efbdf04019dd17 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:35:49 -0400 Subject: [PATCH 5/8] test: port Common Lisp extraction suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the sample.lisp fixture and the Common Lisp extraction tests covering packages, classes, defuns, generics, macros, method specializers, inheritance, imports, custom definers, reader conditionals, and docstring rationale typing. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- tests/fixtures/sample.lisp | 70 ++++++++++ tests/test_multilang.py | 257 ++++++++++++++++++++++++++++++++++++- 2 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/sample.lisp diff --git a/tests/fixtures/sample.lisp b/tests/fixtures/sample.lisp new file mode 100644 index 000000000..4dc2dc417 --- /dev/null +++ b/tests/fixtures/sample.lisp @@ -0,0 +1,70 @@ +(defpackage :http-server + (:use :cl :alexandria) + (:export #:make-server #:start #:stop)) + +(in-package :http-server) + +(deftype port-number () '(integer 1 65535)) + +(defstruct request-stats + bytes-in + bytes-out + duration-ms) + +(defstruct (connection (:conc-name conn-)) + id + socket + state) + +(defvar *active-connections* nil) +(defparameter *default-port* 8080) +(defconstant +max-headers+ 100) + +(define-condition server-error (error) + ((reason :initarg :reason :reader error-reason))) + +(defclass server () + ((host :initarg :host :accessor server-host) + (port :initarg :port :accessor server-port) + (handler :initarg :handler :accessor server-handler))) + +(defclass ssl-server (server) + ((cert-path :initarg :cert-path :accessor ssl-cert-path))) + +(defgeneric process-request (server request)) + +(defmethod process-request ((srv server) (req string)) + "Process an incoming HTTP request." + (let ((parsed (parse-headers req))) + (funcall (server-handler srv) parsed))) + +;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic +(definline-maybe header= (a b) + "Fast header equality." + (string-equal a b)) + +(definline header< (a b) + (string< a b)) + +(defun make-server (host port handler) + "Create a new server instance." + (make-instance 'server :host host :port port :handler handler)) + +(defun start (server) + "Start the server listening on its configured port." + (format t "Starting server on ~a:~a~%" (server-host server) (server-port server)) + (process-request server "GET / HTTP/1.1")) + +(defun stop (server) + (format t "Stopping server~%")) + +(defun compare-headers (h1 h2) + "Compare two headers using the custom definers." + (or (header= h1 h2) (header< h1 h2))) + +(defmacro with-server ((var host port handler) &body body) + "Execute body with a running server bound to var." + `(let ((,var (make-server ,host ,port ,handler))) + (unwind-protect + (progn (start ,var) ,@body) + (stop ,var)))) diff --git a/tests/test_multilang.py b/tests/test_multilang.py index c30b9e10c..2759125d9 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path import pytest -from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql +from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql, extract_commonlisp FIXTURES = Path(__file__).parent / "fixtures" @@ -487,3 +487,258 @@ def test_sql_schema_qualified_alter_fk(): for e in fk_edges: assert e["source"] in node_ids, f"dangling source: {e['source']}" assert e["target"] in node_ids, f"dangling target: {e['target']}" + + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _call_pairs(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _call_pairs(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() + + +# ── extract() dispatch ──────────────────────────────────────────────────────── + +def test_extract_dispatches_all_languages(): + files = [ + FIXTURES / "sample.py", + FIXTURES / "sample.ts", + FIXTURES / "sample.go", + FIXTURES / "sample.rs", + FIXTURES / "sample.lisp", + ] + r = extract(files) + source_files = {n["source_file"] for n in r["nodes"] if n["source_file"]} + assert any("sample.py" in f for f in source_files) + assert any("sample.ts" in f for f in source_files) + assert any("sample.go" in f for f in source_files) + assert any("sample.rs" in f for f in source_files) + assert any("sample.lisp" in f for f in source_files) + + +# ── Cache ───────────────────────────────────────────────────────────────────── + +def test_cache_hit_returns_same_result(tmp_path): + src = FIXTURES / "sample.py" + dst = tmp_path / "sample.py" + dst.write_bytes(src.read_bytes()) + + r1 = extract([dst]) + r2 = extract([dst]) + assert len(r1["nodes"]) == len(r2["nodes"]) + assert len(r1["edges"]) == len(r2["edges"]) + +def test_cache_miss_after_file_change(tmp_path): + dst = tmp_path / "a.py" + dst.write_text("def foo(): pass\n") + r1 = extract([dst]) + + dst.write_text("def foo(): pass\ndef bar(): pass\n") + r2 = extract([dst]) + # bar() should appear in the second result + labels2 = [n["label"] for n in r2["nodes"]] + assert any("bar" in l for l in labels2) From 6a8860e879bc4f10761c98e31979578b8c9b32d5 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:49:40 -0400 Subject: [PATCH 6/8] fix(extract): silence the Common Lisp grammar's deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest tree-sitter-commonlisp release returns an int pointer that the tree-sitter runtime only accepts via a deprecated path, so extracting Lisp spewed a DeprecationWarning per file. Suppress that one warning at the construction site so Lisp extraction runs clean. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- graphify/extract.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index edc6ca504..2d9fd2999 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7,6 +7,7 @@ import re import sys import unicodedata +import warnings from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable @@ -11008,7 +11009,17 @@ def extract_commonlisp(path: Path) -> dict: return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} try: - language = Language(tscl.language()) + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) parser = Parser(language) source = path.read_bytes() tree = parser.parse(source) From 045f6416a9293a363e350d3e08335ab3139e13af Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:58:00 -0400 Subject: [PATCH 7/8] test: house Common Lisp tests with the other language extractors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the Common Lisp extraction tests into the language-extractor suite where the project keeps per-language coverage, so they sit with their peers rather than in the general multi-language file. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- tests/test_languages.py | 214 ++++++++++++++++++++++++++++++++- tests/test_multilang.py | 257 +--------------------------------------- 2 files changed, 214 insertions(+), 257 deletions(-) diff --git a/tests/test_languages.py b/tests/test_languages.py index c8e2acca7..fd676b7f7 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, + extract_powershell, extract_apex, extract_commonlisp, ) FIXTURES = Path(__file__).parent / "fixtures" @@ -1628,3 +1628,215 @@ def test_apex_no_dangling_edges(): for e in r["edges"]: assert e["source"] in node_ids, f"dangling source in {fixture}: {e}" assert e["target"] in node_ids, f"dangling target in {fixture}: {e}" + + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 2759125d9..c30b9e10c 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path import pytest -from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql, extract_commonlisp +from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql FIXTURES = Path(__file__).parent / "fixtures" @@ -487,258 +487,3 @@ def test_sql_schema_qualified_alter_fk(): for e in fk_edges: assert e["source"] in node_ids, f"dangling source: {e['source']}" assert e["target"] in node_ids, f"dangling target: {e['target']}" - - -# ── Common Lisp ────────────────────────────────────────────────────────────── - -def test_cl_finds_package(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "error" not in r - assert "http-server" in _labels(r) - -def test_cl_finds_class(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "server" in _labels(r) - assert "ssl-server" in _labels(r) - -def test_cl_finds_defun(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert any("make-server" in l for l in labels) - assert any("start" in l for l in labels) - assert any("stop" in l for l in labels) - -def test_cl_finds_generic(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert any("process-request" in l for l in _labels(r)) - -def test_cl_finds_macro(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert any("with-server" in l and "macro" in l for l in _labels(r)) - -def test_cl_emits_calls(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - calls = _call_pairs(r) - # start() calls process-request - assert any("start" in src and "process-request" in tgt for src, tgt in calls) - # with-server macro calls make-server and start - assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) - assert any("with-server" in src and "start" in tgt for src, tgt in calls) - -def test_cl_calls_are_extracted(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - for e in r["edges"]: - if e["relation"] == "calls": - assert e["confidence"] == "EXTRACTED" - -def test_cl_no_dangling_edges(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - node_ids = {n["id"] for n in r["nodes"]} - for e in r["edges"]: - if e["relation"] in ("contains", "method", "calls"): - assert e["source"] in node_ids - -def test_cl_docstrings(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] - assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings - labels = _labels(r) - assert any("Process an incoming" in l for l in labels) - -def test_cl_method_specializers(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] - assert len(spec_edges) >= 1 - # process-request specializes on server - assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) - -def test_cl_inherits(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] - assert len(inherit_edges) >= 1 - assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) - -def test_cl_imports(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - import_edges = [e for e in r["edges"] if e["relation"] == "imports"] - targets = {e["target"] for e in import_edges} - assert "cl" in targets - assert "alexandria" in targets - -def test_cl_finds_deftype(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "port-number" in _labels(r) - -def test_cl_finds_defstruct(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert "request-stats" in labels - # defstruct with options form: (defstruct (name ...) ...) - assert "connection" in labels - -def test_cl_finds_defvar_defparameter_defconstant(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert "*active-connections*" in labels - assert "*default-port*" in labels - assert "+max-headers+" in labels - -def test_cl_finds_define_condition(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "server-error" in _labels(r) - -def test_cl_finds_custom_definer(): - """The def-prefix heuristic should catch definline / definline-maybe.""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - # definline-maybe and definline should produce function-style nodes - assert "header=()" in labels - assert "header<()" in labels - -def test_cl_custom_definer_in_call_graph(): - """Functions defined via custom definers should appear in the call graph.""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - calls = _call_pairs(r) - # compare-headers calls header= and header< (defined via definline-maybe / definline) - assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) - assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) - -def test_cl_operator_names_disambiguated(): - """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - # header= and header< must have different ids - eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] - lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] - assert len(eq_ids) == 1 - assert len(lt_ids) == 1 - assert eq_ids[0] != lt_ids[0] - -def test_cl_default_value_not_treated_as_definition(): - """The def-prefix heuristic must not match denylisted symbols.""" - import tempfile - code = "(in-package :cl-user)\n(default-value foo)\n" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - # default-value is denylisted, shouldn't create a "foo" node - assert "foo" not in labels - assert "foo()" not in labels - finally: - path.unlink() - -def test_cl_defs_inside_wrapper_macro(): - """Definitions nested inside wrapper macros like (optimizing ...) or - (eval-when ...) must be extracted. Many CL codebases wrap hot-path - inline functions in application-specific macros.""" - import tempfile - code = """(in-package :cl-user) -(optimizing - (definline-maybe packet-type (p) (aref p 0)) - (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) -(eval-when (:compile-toplevel :load-toplevel :execute) - (defun helper () 42)) -(progn - (defun progn-def () 'ok)) -""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "packet-type()" in labels - assert "set-packet-type()" in labels - assert "helper()" in labels - assert "progn-def()" in labels - finally: - path.unlink() - -def test_cl_defs_inside_reader_conditional(): - """#+feature / #-feature reader conditionals wrap their guarded form - in an include_reader_macro AST node, which the walker must descend - into to find the nested definition.""" - import tempfile - code = """(in-package :cl-user) -#+little-endian -(definline-maybe byte-hash= (a b) (eq a b)) -#-sbcl -(defun only-on-non-sbcl () 1) -""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "byte-hash=()" in labels - assert "only-on-non-sbcl()" in labels - finally: - path.unlink() - -def test_cl_defparameter_string_value_not_docstring(): - """For defvar/defparameter/defconstant, a string literal in the VALUE - position must not be wrongly captured as a docstring node.""" - import tempfile - code = '''(in-package :cl-user) -(defparameter *config-path* "/etc/app/config") -(defvar *greeting* "hello world") -''' - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "*config-path*" in labels - assert "*greeting*" in labels - # The string VALUES must not show up as rationale nodes - assert not any("/etc/app/config" in l for l in labels) - assert not any("hello world" in l for l in labels) - finally: - path.unlink() - - -# ── extract() dispatch ──────────────────────────────────────────────────────── - -def test_extract_dispatches_all_languages(): - files = [ - FIXTURES / "sample.py", - FIXTURES / "sample.ts", - FIXTURES / "sample.go", - FIXTURES / "sample.rs", - FIXTURES / "sample.lisp", - ] - r = extract(files) - source_files = {n["source_file"] for n in r["nodes"] if n["source_file"]} - assert any("sample.py" in f for f in source_files) - assert any("sample.ts" in f for f in source_files) - assert any("sample.go" in f for f in source_files) - assert any("sample.rs" in f for f in source_files) - assert any("sample.lisp" in f for f in source_files) - - -# ── Cache ───────────────────────────────────────────────────────────────────── - -def test_cache_hit_returns_same_result(tmp_path): - src = FIXTURES / "sample.py" - dst = tmp_path / "sample.py" - dst.write_bytes(src.read_bytes()) - - r1 = extract([dst]) - r2 = extract([dst]) - assert len(r1["nodes"]) == len(r2["nodes"]) - assert len(r1["edges"]) == len(r2["edges"]) - -def test_cache_miss_after_file_change(tmp_path): - dst = tmp_path / "a.py" - dst.write_text("def foo(): pass\n") - r1 = extract([dst]) - - dst.write_text("def foo(): pass\ndef bar(): pass\n") - r2 = extract([dst]) - # bar() should appear in the second result - labels2 = [n["label"] for n in r2["nodes"]] - assert any("bar" in l for l in labels2) From cd1b6ae919e2b6b04573b037c8b36148f9a9cc57 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:58:01 -0400 Subject: [PATCH 8/8] docs: list Common Lisp in the README language and extras tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface Common Lisp in the supported-extensions table and add the commonlisp optional-extra row so users know the grammar exists and how to install it. 🄯 Brian O'Reilly , 2026 Co-authored-by: jansaasman --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b7ca37251..fabc6309a 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ Install only what you need: | `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | | `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | +| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | | `all` | Everything above | `uv tool install "graphifyy[all]"` | @@ -237,7 +238,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (28 tree-sitter grammars) | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .h .hpp .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .csproj .fsproj .vbproj .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`) | +| Code (29 tree-sitter grammars) | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .h .hpp .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .csproj .fsproj .vbproj .razor .cshtml .lisp .cl .lsp .asd` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.lisp`/`.cl`/`.lsp`/`.asd` requires `uv tool install graphifyy[commonlisp]`) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements |