From 385600020e4fd8b712a5297ebc10a1464eb4a874 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Tue, 22 Sep 2026 14:05:57 +1000 Subject: [PATCH 1/4] feat(harness): fail a project Skill that points at a path it invented An Agent cannot verify its own file pointers. A project Skill that names `src/server/routes.ts` reads exactly like one that names a real file, and the Agent has no way to tell the difference after the fact. The Harness already prepares the project source, so it knows every path the Agent could have seen. Anything else in an inline code span is invention, and promotion now fails with the path named. A line that reports a path as missing passes. "The project has no `src/lib.rs`" is a fact about the project, not a pointer. Adds `evals/`, which scores the same Skills on the direct path, where no check runs. Three cases, each run with and without the Skills loaded. --- .claude-plugin/plugin.json | 8 + .gitignore | 1 + CLAUDE.md | 9 + CONTEXT.md | 6 + evals/README.md | 53 ++++++ evals/package-not-project/case.yaml | 17 ++ evals/package-not-project/graders/criteria.md | 20 ++ .../graders/skill-fired.md | 6 + .../graders/skill-written.md | 5 + evals/package-not-project/scaffold.sh | 17 ++ evals/project-skill-real-paths/case.yaml | 16 ++ .../graders/criteria.md | 27 +++ .../graders/search-command.md | 7 + .../graders/skill-fired.md | 6 + .../graders/skill-written.md | 5 + evals/project-skill-real-paths/scaffold.sh | 23 +++ evals/project-skill-rust/case.yaml | 17 ++ evals/project-skill-rust/graders/criteria.md | 26 +++ .../graders/search-command.md | 7 + .../project-skill-rust/graders/skill-fired.md | 6 + .../graders/skill-written.md | 5 + evals/project-skill-rust/scaffold.sh | 20 ++ packages/harness/src/harness.ts | 8 +- .../harness/src/internal/output/project.ts | 110 +++++++++++ .../harness/src/internal/output/validate.ts | 9 + packages/harness/test/e2e/harness.test.ts | 6 +- .../harness/test/e2e/project-skill.test.ts | 178 ++++++++++++++++++ .../test/e2e/promotion-cleanup.test.ts | 6 +- packages/harness/test/support/fakes.ts | 16 +- scripts/eval-skills.sh | 35 ++++ 30 files changed, 667 insertions(+), 8 deletions(-) create mode 100644 .claude-plugin/plugin.json create mode 100644 evals/README.md create mode 100644 evals/package-not-project/case.yaml create mode 100644 evals/package-not-project/graders/criteria.md create mode 100644 evals/package-not-project/graders/skill-fired.md create mode 100644 evals/package-not-project/graders/skill-written.md create mode 100755 evals/package-not-project/scaffold.sh create mode 100644 evals/project-skill-real-paths/case.yaml create mode 100644 evals/project-skill-real-paths/graders/criteria.md create mode 100644 evals/project-skill-real-paths/graders/search-command.md create mode 100644 evals/project-skill-real-paths/graders/skill-fired.md create mode 100644 evals/project-skill-real-paths/graders/skill-written.md create mode 100755 evals/project-skill-real-paths/scaffold.sh create mode 100644 evals/project-skill-rust/case.yaml create mode 100644 evals/project-skill-rust/graders/criteria.md create mode 100644 evals/project-skill-rust/graders/search-command.md create mode 100644 evals/project-skill-rust/graders/skill-fired.md create mode 100644 evals/project-skill-rust/graders/skill-written.md create mode 100755 evals/project-skill-rust/scaffold.sh create mode 100644 packages/harness/src/internal/output/project.ts create mode 100644 packages/harness/test/e2e/project-skill.test.ts create mode 100755 scripts/eval-skills.sh diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 00000000..e7dafd13 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "skilld", + "description": "The skilld-maintained Skills for Skill generation, review, and skilld CLI use.", + "version": "3.1.0", + "experimental": { + "evals": "evals" + } +} diff --git a/.gitignore b/.gitignore index 259db9f7..156bac16 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ coverage .artifacts .guides-out +evals/results diff --git a/CLAUDE.md b/CLAUDE.md index 9031a80a..8b3abb5f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,6 +22,15 @@ cargo test --workspace cargo clippy --workspace --all-targets -- -D warnings ``` +Score the skilld-maintained Skills against scaffolded projects: + +```sh +./scripts/eval-skills.sh +``` + +Every case runs a real Agent session and costs money. Read `evals/README.md` +before changing a case or a grader. + ## Product boundary The native `skilld` CLI searches, runs, installs, lists, views, removes, updates, and verifies Skills. diff --git a/CONTEXT.md b/CONTEXT.md index 9bd8cc0a..fa8a28c3 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -18,6 +18,12 @@ Direct runs remain user reviewed. Harness runs apply deterministic checks before atomic promotion. +A project Skill must point at prepared project files. + +The Harness rejects a pointer the prepared source does not contain. + +`evals` scores the same Skills on the direct path, where no check runs. + ## Trust - `verified` records an Artifact with a valid attestation. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 00000000..325eea66 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,53 @@ +# skilld-maintained Skill evals + +These cases score the Skills in `skills/` against scaffolded projects. + +Direct Skill runs have no enforcement. An Agent reads the Skill and writes files. +These cases measure what the Agent actually produces. + +## Run them + +```sh +./scripts/eval-skills.sh +./scripts/eval-skills.sh --case project-skill-rust --runs 1 +``` + +Each run is a real Agent session on your own credential. The suite costs money, +so it never runs in `pnpm test` or in CI on a pull request. + +The script stages `skills/`, `evals/`, and `.claude-plugin/` into a temporary +directory before it runs. The repository holds `target/` and `node_modules/`, +which overflow the eval's argument list. + +## Read the score + +Every case runs twice: once with the Skills loaded, once without. The delta is +the value the Skills add. A case that scores the same in both arms measures the +model, not the Skill. + +Recorded on 2026-09-22, 2 runs per arm, Claude Code 2.1.278: + +| Case | With | Without | Δ | +| --- | --- | --- | --- | +| `package-not-project` | 1.00 | 0.67 | +0.33 | +| `project-skill-real-paths` | 1.00 | 0.75 | +0.25 | +| `project-skill-rust` | 1.00 | 0.75 | +0.25 | + +## What each case holds + +- `project-skill-real-paths`: a TypeScript project with a `dist/` decoy. Checks + that the Skill names real paths and gives a search the Agent can repeat. +- `project-skill-rust`: a Cargo project with no `package.json`. Checks that the + Skill reads the project's own manifest and finds its declared binary. +- `package-not-project`: a published package. Checks that a request for consumer + instructions routes to `generate-package-skill`. + +## Write a case + +A grader reads the transcript, not the files the Agent wrote. Ask the prompt for +a closing summary that states what a grader needs to see. + +An `llm` grader judges shape and intent. A `regex` or `file_exists` grader +settles a fact. Deterministic rules about SKILL.md content belong in the Harness +instead, where `packages/harness/src/internal/output/project.ts` fails a Skill +before promotion. diff --git a/evals/package-not-project/case.yaml b/evals/package-not-project/case.yaml new file mode 100644 index 00000000..fbef83db --- /dev/null +++ b/evals/package-not-project/case.yaml @@ -0,0 +1,17 @@ +schema_version: '1.0' +name: package-not-project +description: A Skill for a package's consumers uses the package Skill path, not the project one. +tags: [project-skill, routing] +runs: 2 +context: + scaffold_script: ./scaffold.sh +execution: + max_turns: 30 + allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit] + prompt: | + I maintain the `tinyfmt` package published on npm. Draft an Agent Skill that + teaches an Agent to use `tinyfmt` in someone else's codebase. Write it to + `skills/tinyfmt/SKILL.md`. + + End with a short summary that says which skilld-maintained Skill you followed + and why. diff --git a/evals/package-not-project/graders/criteria.md b/evals/package-not-project/graders/criteria.md new file mode 100644 index 00000000..d94240f1 --- /dev/null +++ b/evals/package-not-project/graders/criteria.md @@ -0,0 +1,20 @@ +--- +type: llm +weight: 2 +focus: last_message +--- + +The request is for a Skill that teaches consumers to use the `tinyfmt` package, +not for a Skill that navigates this workspace. + +A passing run: + +- Follows `generate-package-skill`, and says so. +- Writes consumer-facing instructions: the public export `formatMoney`, its + arguments, and how to import it. +- Does not turn the result into a project navigation Skill. A layout table of + this repository, or search commands for changing this repository, is the wrong + shape here. + +It fails if it follows `generate-project-skill`, or if the Skill it writes reads +as internal navigation for this workspace rather than usage for a consumer. diff --git a/evals/package-not-project/graders/skill-fired.md b/evals/package-not-project/graders/skill-fired.md new file mode 100644 index 00000000..5b465cad --- /dev/null +++ b/evals/package-not-project/graders/skill-fired.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +min: 1 +arm: with-only +--- diff --git a/evals/package-not-project/graders/skill-written.md b/evals/package-not-project/graders/skill-written.md new file mode 100644 index 00000000..ab722b0e --- /dev/null +++ b/evals/package-not-project/graders/skill-written.md @@ -0,0 +1,5 @@ +--- +type: file_exists +path: skills/tinyfmt/SKILL.md +exists: true +--- diff --git a/evals/package-not-project/scaffold.sh b/evals/package-not-project/scaffold.sh new file mode 100755 index 00000000..70dee6b4 --- /dev/null +++ b/evals/package-not-project/scaffold.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e +mkdir -p src test +cat > package.json <<'JSON' +{ + "name": "tinyfmt", + "description": "Format a number as a currency string.", + "type": "module", + "version": "1.4.0", + "exports": "./src/index.js", + "types": "./src/index.d.ts" +} +JSON +printf '# tinyfmt\n\nFormat a number as a currency string.\n\n## Usage\n\n```js\nimport { formatMoney } from "tinyfmt"\n```\n' > README.md +printf 'export const formatMoney = (cents, currency = "USD") => new Intl.NumberFormat("en-US", { style: "currency", currency }).format(cents / 100)\n' > src/index.js +printf 'export declare const formatMoney: (cents: number, currency?: string) => string\n' > src/index.d.ts +printf 'import { formatMoney } from "../src/index.js"\nconsole.log(formatMoney(1234))\n' > test/format.test.js diff --git a/evals/project-skill-real-paths/case.yaml b/evals/project-skill-real-paths/case.yaml new file mode 100644 index 00000000..cdde7108 --- /dev/null +++ b/evals/project-skill-real-paths/case.yaml @@ -0,0 +1,16 @@ +schema_version: '1.0' +name: project-skill-real-paths +description: A project Skill drafted for this workspace points only at files the project contains. +tags: [project-skill] +runs: 2 +context: + scaffold_script: ./scaffold.sh +execution: + max_turns: 30 + allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit] + prompt: | + Draft an Agent Skill for this project so a future session can navigate it + without reading every file first. Write it to `skills/invoicer/SKILL.md`. + + End with a short summary that quotes one search command from the Skill and + lists every project path the Skill names. diff --git a/evals/project-skill-real-paths/graders/criteria.md b/evals/project-skill-real-paths/graders/criteria.md new file mode 100644 index 00000000..f63f7e76 --- /dev/null +++ b/evals/project-skill-real-paths/graders/criteria.md @@ -0,0 +1,27 @@ +--- +type: llm +weight: 2 +focus: last_message +--- + +The run drafted a project Skill for the `invoicer` workspace. + +A passing run: + +- Names only paths the project contains. These exist: `package.json`, + `README.md`, `src/index.ts`, `src/core/render.ts`, `src/core/send.ts`, + `src/commands/send.ts`, `docs/ledger.md`, `dist/index.js`, `dist/send.js`. + Any other source path is invented. +- Names the entry point the manifest declares, `src/index.ts`. +- Gives at least one search command the Agent can repeat, such as an `rg` + invocation scoped to a real directory. +- Treats `dist/` as generated output. Naming it as a directory to leave alone + passes. Pointing at it as a place to read or edit source fails. +- Uses project-relative paths, not absolute ones. + +Naming a path the project lacks passes when the Skill says it is missing. A +report such as "there is no `src/lib.rs`" is a fact about the project. +Presenting a path that does not exist as a real file is what fails. + +It fails if it names a path the project does not contain, sends the Agent into +`dist/` to read source, or gives no repeatable search command. diff --git a/evals/project-skill-real-paths/graders/search-command.md b/evals/project-skill-real-paths/graders/search-command.md new file mode 100644 index 00000000..51db69ec --- /dev/null +++ b/evals/project-skill-real-paths/graders/search-command.md @@ -0,0 +1,7 @@ +--- +type: regex +target: last_message +pattern: "(rg|grep) " +match: contains +weight: 1 +--- diff --git a/evals/project-skill-real-paths/graders/skill-fired.md b/evals/project-skill-real-paths/graders/skill-fired.md new file mode 100644 index 00000000..5b465cad --- /dev/null +++ b/evals/project-skill-real-paths/graders/skill-fired.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +min: 1 +arm: with-only +--- diff --git a/evals/project-skill-real-paths/graders/skill-written.md b/evals/project-skill-real-paths/graders/skill-written.md new file mode 100644 index 00000000..8fbae776 --- /dev/null +++ b/evals/project-skill-real-paths/graders/skill-written.md @@ -0,0 +1,5 @@ +--- +type: file_exists +path: skills/invoicer/SKILL.md +exists: true +--- diff --git a/evals/project-skill-real-paths/scaffold.sh b/evals/project-skill-real-paths/scaffold.sh new file mode 100755 index 00000000..daee397c --- /dev/null +++ b/evals/project-skill-real-paths/scaffold.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e +mkdir -p src/commands src/core docs dist +cat > package.json <<'JSON' +{ + "name": "invoicer", + "description": "Render and send invoices from a ledger file.", + "type": "module", + "main": "./src/index.ts", + "scripts": { + "test": "vitest run", + "lint": "eslint ." + } +} +JSON +printf '# invoicer\n\nRender and send invoices from a ledger file.\n' > README.md +printf 'export { renderInvoice } from "./core/render.ts"\nexport { sendInvoice } from "./core/send.ts"\n' > src/index.ts +printf 'export const renderInvoice = (ledger: string): string => ledger.trim()\n' > src/core/render.ts +printf 'export const sendInvoice = async (body: string): Promise => { void body }\n' > src/core/send.ts +printf 'export const runSend = async (): Promise => {}\n' > src/commands/send.ts +printf '# Ledger format\n\nOne invoice per line.\n' > docs/ledger.md +printf 'export const renderInvoice=(l)=>l.trim();\n' > dist/index.js +printf 'export const sendInvoice=async(b)=>{};\n' > dist/send.js diff --git a/evals/project-skill-rust/case.yaml b/evals/project-skill-rust/case.yaml new file mode 100644 index 00000000..2a2881fe --- /dev/null +++ b/evals/project-skill-rust/case.yaml @@ -0,0 +1,17 @@ +schema_version: '1.0' +name: project-skill-rust +description: A project Skill reads the project's own manifest, not a JavaScript one, and finds its real entry point. +tags: [project-skill] +runs: 2 +context: + scaffold_script: ./scaffold.sh +execution: + max_turns: 30 + allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit] + prompt: | + Draft an Agent Skill for this project so a future session can navigate it + without reading every file first. Write it to `skills/ledgerd/SKILL.md`. + + End with a short summary that quotes one search command from the Skill and + lists every project path the Skill names and + the entry point it found. diff --git a/evals/project-skill-rust/graders/criteria.md b/evals/project-skill-rust/graders/criteria.md new file mode 100644 index 00000000..fd1843fe --- /dev/null +++ b/evals/project-skill-rust/graders/criteria.md @@ -0,0 +1,26 @@ +--- +type: llm +weight: 2 +focus: last_message +--- + +The run drafted a project Skill for the `ledgerd` Rust workspace. + +A passing run: + +- Reads `Cargo.toml` for the project name and entry point. It never claims a + `package.json`, which this project does not have. +- Names `src/main.rs` as the binary entry point, as `[[bin]]` declares. +- Names only paths the project contains: `Cargo.toml`, `README.md`, + `src/main.rs`, `src/store/mod.rs`, `src/store/journal.rs`, `tests/journal.rs`, + `target/debug/ledgerd`. Any other source path is invented. +- Gives at least one search command the Agent can repeat, scoped to a real + directory such as `src`. +- Treats `target/` as generated output rather than source to read. + +Naming a path the project lacks passes when the Skill says it is missing. A +report such as "there is no `src/lib.rs`" is a fact about the project. +Presenting a path that does not exist as a real file is what fails. + +It fails if it invents a path, claims a manifest the project does not have, +names the wrong entry point, or gives no repeatable search command. diff --git a/evals/project-skill-rust/graders/search-command.md b/evals/project-skill-rust/graders/search-command.md new file mode 100644 index 00000000..51db69ec --- /dev/null +++ b/evals/project-skill-rust/graders/search-command.md @@ -0,0 +1,7 @@ +--- +type: regex +target: last_message +pattern: "(rg|grep) " +match: contains +weight: 1 +--- diff --git a/evals/project-skill-rust/graders/skill-fired.md b/evals/project-skill-rust/graders/skill-fired.md new file mode 100644 index 00000000..5b465cad --- /dev/null +++ b/evals/project-skill-rust/graders/skill-fired.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +min: 1 +arm: with-only +--- diff --git a/evals/project-skill-rust/graders/skill-written.md b/evals/project-skill-rust/graders/skill-written.md new file mode 100644 index 00000000..10b8eebf --- /dev/null +++ b/evals/project-skill-rust/graders/skill-written.md @@ -0,0 +1,5 @@ +--- +type: file_exists +path: skills/ledgerd/SKILL.md +exists: true +--- diff --git a/evals/project-skill-rust/scaffold.sh b/evals/project-skill-rust/scaffold.sh new file mode 100755 index 00000000..af62b4c0 --- /dev/null +++ b/evals/project-skill-rust/scaffold.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e +mkdir -p src/store tests target/debug +cat > Cargo.toml <<'TOML' +[package] +name = "ledgerd" +version = "0.2.0" +edition = "2021" +description = "Append-only ledger daemon." + +[[bin]] +name = "ledgerd" +path = "src/main.rs" +TOML +printf '# ledgerd\n\nAppend-only ledger daemon.\n' > README.md +printf 'mod store;\n\nfn main() {\n store::open();\n}\n' > src/main.rs +printf 'pub mod journal;\n\npub fn open() {}\n' > src/store/mod.rs +printf 'pub fn append(entry: &str) -> usize {\n entry.len()\n}\n' > src/store/journal.rs +printf '#[test]\nfn appends_an_entry() {\n assert_eq!(ledgerd::store::journal::append("a"), 1);\n}\n' > tests/journal.rs +printf 'binary\n' > target/debug/ledgerd diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts index 5c0883bd..66777842 100644 --- a/packages/harness/src/harness.ts +++ b/packages/harness/src/harness.ts @@ -237,7 +237,13 @@ export function createSkillHarness(options: CreateSkillHarnessOptions): SkillHar if (prepared.value.skillName === 'review-skill') return validateSkillReview(collected.value) - const validated = validateGeneratedSkill(prepared.value.outputName, collected.value) + const validated = validateGeneratedSkill( + prepared.value.outputName, + collected.value, + prepared.value.skillName === 'generate-project-skill' + ? prepared.value.source.files.map(file => file.path) + : undefined, + ) if (validated._tag === 'Err') return validated if (!prepared.value.destination) diff --git a/packages/harness/src/internal/output/project.ts b/packages/harness/src/internal/output/project.ts new file mode 100644 index 00000000..2ef698a2 --- /dev/null +++ b/packages/harness/src/internal/output/project.ts @@ -0,0 +1,110 @@ +const searchCommands = ['rg ', 'grep ', 'git grep '] + +/** Inline code spans, which is where a Skill writes its file pointers. */ +const inlineCode = /`([^`\n]+)`/g + +/** Fenced blocks, which hold commands rather than pointers. */ +const fencedBlock = /```[^\n]*\n[\s\S]*?```/g + +/** + * A Skill may report a file the project lacks, such as a missing entry point. + * A line that denies a path states a fact about the project, not a pointer. + */ +const denial = /\b(no|not|none|never|absent|missing|lacks|without|instead|rather|empty|undeclared|unavailable|nothing)\b|\bdoes ?n[o']t\b/i + +function fencedBlocks(markdown: string): ReadonlyArray { + return markdown.match(fencedBlock) ?? [] +} + +/** Prose lines, with fenced commands removed so a command never reads as a pointer. */ +function proseLines(markdown: string): ReadonlyArray { + return markdown.replace(fencedBlock, '').split('\n') +} + +/** + * A token counts as a file pointer only when it cannot be anything else. + * A false pointer fails a correct Skill, so every rule here excludes rather than includes. + */ +function pathCandidate(token: string): string | null { + if (token.length === 0 || /\s/.test(token)) + return null + // A flag, a variable, a comment, or a placeholder is never a pointer. + if (/^[-$#<{[]/.test(token)) + return null + // A URL, a package specifier, or a shell construct is never a pointer. + if (token.includes('://') || /[()|;&"'@:]/.test(token)) + return null + const trimmed = token.replace(/^\.\//, '').replace(/\/+$/, '') + if (trimmed.length === 0 || trimmed.startsWith('..')) + return null + // A bare word such as `rg` or `pnpm` carries no path shape. + if (!trimmed.includes('/') && !/\.[a-z0-9]+$/i.test(trimmed)) + return null + return trimmed +} + +/** The literal prefix of a glob, which is the part a real path must start with. */ +function literalPrefix(candidate: string): string { + const wildcard = candidate.search(/[*?]/) + if (wildcard === -1) + return candidate + const cut = candidate.slice(0, wildcard).replace(/[^/]*$/, '') + return cut.replace(/\/+$/, '') +} + +function resolves(candidate: string, known: ReadonlySet): boolean { + if (known.has(candidate)) + return true + const prefix = literalPrefix(candidate) + if (prefix.length === 0) + return true + // A directory pointer resolves when the project holds a file under it. + for (const path of known) { + if (path === prefix || path.startsWith(`${prefix}/`)) + return true + } + return false +} + +export interface ProjectSkillInput { + /** The generated SKILL.md text. */ + readonly markdown: string + /** Project-relative paths the Harness prepared for the Agent. */ + readonly projectPaths: ReadonlyArray + /** Output-relative paths the Skill writes beside SKILL.md. */ + readonly outputPaths: ReadonlyArray +} + +/** + * Check that a project Skill points at files the project contains and gives a + * search the Agent can repeat. The Agent cannot verify its own pointers. + * + * The Harness hides generated directories from the Agent, so a pointer at one is + * invention like any other. A line that states a path is missing passes, because + * reporting an absent file is a fact rather than a pointer. + */ +export function checkProjectSkill(input: ProjectSkillInput): ReadonlyArray { + const issues: string[] = [] + const known = new Set([...input.projectPaths, ...input.outputPaths]) + const unknown = new Set() + + for (const line of proseLines(input.markdown)) { + if (denial.test(line)) + continue + for (const match of line.matchAll(inlineCode)) { + const candidate = pathCandidate(match[1]!.trim()) + if (candidate !== null && !resolves(candidate, known)) + unknown.add(candidate) + } + } + + for (const path of [...unknown].sort()) + issues.push(`SKILL.md points at a path the project does not contain: ${path}.`) + + const inline = [...input.markdown.matchAll(inlineCode)].map(match => match[1]!) + const commands = [...fencedBlocks(input.markdown), ...inline].join('\n') + if (!searchCommands.some(command => commands.includes(command))) + issues.push('SKILL.md must give the Agent at least one search command it can repeat.') + + return issues +} diff --git a/packages/harness/src/internal/output/validate.ts b/packages/harness/src/internal/output/validate.ts index 2b121f72..81de3d14 100644 --- a/packages/harness/src/internal/output/validate.ts +++ b/packages/harness/src/internal/output/validate.ts @@ -3,6 +3,7 @@ import type { Result } from '../result.ts' import type { CollectedFile } from './collect.ts' import { parseDocument } from 'yaml' import { isSkillName, normalizeOutputPath } from '../paths.ts' +import { checkProjectSkill } from './project.ts' import { err, ok } from '../result.ts' const allowedFrontmatter = new Set([ @@ -35,6 +36,7 @@ const isStringMap = (value: unknown): boolean => export const validateGeneratedSkill = ( name: string, files: ReadonlyArray, + project?: ReadonlyArray, ): Result => { const issues: string[] = [] const skillFiles = files.filter(file => file.path === 'SKILL.md') @@ -44,6 +46,13 @@ export const validateGeneratedSkill = ( const source = decodeText(skillFiles[0]!.content) if (source === null) return invalid(['SKILL.md must contain valid UTF-8 text.']) + if (project !== undefined) { + issues.push(...checkProjectSkill({ + markdown: source, + projectPaths: project, + outputPaths: files.map(file => file.path), + })) + } const match = source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/) if (!match) return invalid(['SKILL.md must start with YAML frontmatter.']) diff --git a/packages/harness/test/e2e/harness.test.ts b/packages/harness/test/e2e/harness.test.ts index 6dd8c2bd..85f52eee 100644 --- a/packages/harness/test/e2e/harness.test.ts +++ b/packages/harness/test/e2e/harness.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, readFile, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { createSkillHarness } from '../../src/index.ts' -import { createFakeHarness, createFakeSandboxProvider, skillSource } from '../support/fakes.ts' +import { createFakeHarness, createFakeSandboxProvider, projectSkillBody, skillSource } from '../support/fakes.ts' async function makePackage() { const root = await mkdtemp(join(tmpdir(), 'skilld-package-')) @@ -258,7 +258,7 @@ describe('createSkillHarness', () => { const destinationRoot = await mkdtemp(join(tmpdir(), 'skilld-output-')) const currentDir = join(destinationRoot, 'example-project') await mkdir(join(currentDir, 'references'), { recursive: true }) - await writeFile(join(currentDir, 'SKILL.md'), skillSource('example-project', '# Old instructions\n')) + await writeFile(join(currentDir, 'SKILL.md'), skillSource('example-project', projectSkillBody('# Old instructions'))) await writeFile(join(currentDir, 'references/old.md'), 'old reference\n') const fake = createFakeHarness({ async onPrompt({ sandbox, workDir, options }) { @@ -266,7 +266,7 @@ describe('createSkillHarness', () => { expect(promptText(options.prompt)).toContain('/input/current-skill') await sandbox.writeTextFile({ path: join(workDir, 'skilld-output/example-project/SKILL.md'), - content: skillSource('example-project', '# New instructions\n'), + content: skillSource('example-project', projectSkillBody('# New instructions')), }) }, }) diff --git a/packages/harness/test/e2e/project-skill.test.ts b/packages/harness/test/e2e/project-skill.test.ts new file mode 100644 index 00000000..51b1bbb1 --- /dev/null +++ b/packages/harness/test/e2e/project-skill.test.ts @@ -0,0 +1,178 @@ +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createSkillHarness } from '../../src/index.ts' +import { createFakeHarness, createFakeSandboxProvider } from '../support/fakes.ts' + +async function makeProject(): Promise { + const root = await mkdtemp(join(tmpdir(), 'skilld-project-nav-')) + await writeFile(join(root, 'package.json'), '{"name":"example-project","main":"./src/index.ts"}\n') + await writeFile(join(root, 'README.md'), '# Example project\n') + await mkdir(join(root, 'src')) + await writeFile(join(root, 'src/index.ts'), 'export const start = () => true\n') + await mkdir(join(root, 'dist')) + await writeFile(join(root, 'dist/index.js'), 'export const start=()=>true\n') + return root +} + +function projectSkill(body: string): string { + return `---\nname: example-project\ndescription: Work on the example project through its real files.\n---\n\n${body}` +} + +const conforming = [ + '# Example project', + '', + 'The entry point is `src/index.ts`. The manifest is `package.json`.', + '', + 'Search the source:', + '', + '```sh', + 'rg -n "export " src', + '```', + '', +].join('\n') + +async function runProjectSkill(markdown: string): Promise<{ + result: Awaited['run']>> + destinationRoot: string +}> { + const projectDir = await makeProject() + const destinationRoot = await mkdtemp(join(tmpdir(), 'skilld-project-nav-out-')) + const { harness } = createFakeHarness({ + async onPrompt({ sandbox, workDir }) { + await sandbox.writeTextFile({ + path: join(workDir, 'skilld-output/example-project/SKILL.md'), + content: markdown, + }) + }, + }) + const result = await createSkillHarness({ harness, sandbox: createFakeSandboxProvider() }).run({ + _tag: 'ProjectSkill', + projectDir, + destination: { rootDir: destinationRoot, name: 'example-project' }, + }) + return { result, destinationRoot } +} + +describe('project Skill navigation checks', () => { + it('promotes a project Skill whose pointers and searches hold', async () => { + const { result, destinationRoot } = await runProjectSkill(projectSkill(conforming)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + await expect(readFile(join(destinationRoot, 'example-project/SKILL.md'), 'utf8')).resolves.toContain('src/index.ts') + }) + + it('refuses a pointer the project does not contain', async () => { + const body = conforming.replace('`src/index.ts`', '`src/server/boot.ts`') + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result._tag).toBe('Err') + if (result._tag !== 'Err' || result.error._tag !== 'InvalidSkill') + throw new Error('Expected an InvalidSkill error.') + expect(result.error.issues).toContain('SKILL.md points at a path the project does not contain: src/server/boot.ts.') + }) + + it('accepts a path a line reports as missing', async () => { + const body = conforming.replace( + 'The entry point is `src/index.ts`.', + 'The entry point is `src/index.ts`. The project has no `src/lib.ts`, so the test cannot import one.', + ) + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + + it('accepts a generated path a line puts off limits', async () => { + const body = conforming.replace( + 'The entry point is `src/index.ts`.', + 'The entry point is `src/index.ts`. Never edit `dist/index.js`, which the build writes.', + ) + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + + it('refuses an invented path a line presents as real', async () => { + const body = conforming.replace( + 'The entry point is `src/index.ts`.', + 'The entry point is `src/index.ts`. Request handling lives in `src/server/routes.ts`.', + ) + + const { result } = await runProjectSkill(projectSkill(body)) + + if (result._tag !== 'Err' || result.error._tag !== 'InvalidSkill') + throw new Error('Expected an InvalidSkill error.') + expect(result.error.issues).toContain('SKILL.md points at a path the project does not contain: src/server/routes.ts.') + }) + + it('refuses a project Skill that gives the Agent no search command', async () => { + const body = ['# Example project', '', 'The entry point is `src/index.ts`.', ''].join('\n') + + const { result } = await runProjectSkill(projectSkill(body)) + + if (result._tag !== 'Err' || result.error._tag !== 'InvalidSkill') + throw new Error('Expected an InvalidSkill error.') + expect(result.error.issues).toContain('SKILL.md must give the Agent at least one search command it can repeat.') + }) + + it('accepts a directory pointer, a glob, and a file it writes beside SKILL.md', async () => { + const body = [ + '# Example project', + '', + 'Source lives in `src/`. Types live in `src/**/*.ts`.', + 'Architecture notes are in `references/architecture.md`.', + '', + '```sh', + 'rg -n "export " src', + '```', + '', + ].join('\n') + const projectDir = await makeProject() + const destinationRoot = await mkdtemp(join(tmpdir(), 'skilld-project-nav-out-')) + const { harness } = createFakeHarness({ + async onPrompt({ sandbox, workDir }) { + await sandbox.writeTextFile({ + path: join(workDir, 'skilld-output/example-project/SKILL.md'), + content: projectSkill(body), + }) + await sandbox.writeTextFile({ + path: join(workDir, 'skilld-output/example-project/references/architecture.md'), + content: '# Architecture\n', + }) + }, + }) + + const result = await createSkillHarness({ harness, sandbox: createFakeSandboxProvider() }).run({ + _tag: 'ProjectSkill', + projectDir, + destination: { rootDir: destinationRoot, name: 'example-project' }, + }) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + + it('leaves a package Skill unchecked for project navigation', async () => { + const projectDir = await makeProject() + const destinationRoot = await mkdtemp(join(tmpdir(), 'skilld-package-nav-out-')) + const body = ['# Example package', '', 'Read `src/server/boot.ts` for the entry point.', ''].join('\n') + const { harness } = createFakeHarness({ + async onPrompt({ sandbox, workDir }) { + await sandbox.writeTextFile({ + path: join(workDir, 'skilld-output/example-package/SKILL.md'), + content: `---\nname: example-package\ndescription: Use the example package through its public API.\n---\n\n${body}`, + }) + }, + }) + + const result = await createSkillHarness({ harness, sandbox: createFakeSandboxProvider() }).run({ + _tag: 'PackageSkill', + source: { _tag: 'LocalPackage', rootDir: projectDir, packageDir: '.' }, + destination: { rootDir: destinationRoot, name: 'example-package' }, + }) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) +}) diff --git a/packages/harness/test/e2e/promotion-cleanup.test.ts b/packages/harness/test/e2e/promotion-cleanup.test.ts index 5375bb04..88823851 100644 --- a/packages/harness/test/e2e/promotion-cleanup.test.ts +++ b/packages/harness/test/e2e/promotion-cleanup.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, vi } from 'vitest' import { createSkillHarness } from '../../src/index.ts' -import { createFakeHarness, createFakeSandboxProvider, skillSource } from '../support/fakes.ts' +import { createFakeHarness, createFakeSandboxProvider, projectSkillBody, skillSource } from '../support/fakes.ts' const failures = vi.hoisted(() => ({ backup: false, lock: false, promotion: false })) @@ -37,12 +37,12 @@ async function runReplacement() { const currentDir = join(destinationRoot, 'example-project') await writeFile(join(projectDir, 'package.json'), '{}\n') await mkdir(currentDir) - await writeFile(join(currentDir, 'SKILL.md'), skillSource('example-project', '# Old\n')) + await writeFile(join(currentDir, 'SKILL.md'), skillSource('example-project', projectSkillBody('# Old'))) const fake = createFakeHarness({ async onPrompt({ sandbox, workDir }) { await sandbox.writeTextFile({ path: join(workDir, 'skilld-output/example-project/SKILL.md'), - content: skillSource('example-project', '# New\n'), + content: skillSource('example-project', projectSkillBody('# New')), }) }, }) diff --git a/packages/harness/test/support/fakes.ts b/packages/harness/test/support/fakes.ts index dcbd09c3..5fec6c92 100644 --- a/packages/harness/test/support/fakes.ts +++ b/packages/harness/test/support/fakes.ts @@ -201,4 +201,18 @@ export function createFakeSandboxProvider() { } } -export const skillSource = (name: string, body = '# Instructions\n\nUse the package API.\n'): string => `---\nname: ${name}\ndescription: Use ${name} when working with its public API.\n---\n\n${body}` +/** A body that satisfies the project Skill navigation checks. */ +export const projectSkillBody = (heading = '# Instructions'): string => [ + heading, + '', + 'Read `package.json` for the name and the entry point.', + '', + 'Search the source:', + '', + '```sh', + 'rg -n "export " .', + '```', + '', +].join('\n') + +export const skillSource = (name: string, body = projectSkillBody()): string => `---\nname: ${name}\ndescription: Use ${name} when working with its public API.\n---\n\n${body}` diff --git a/scripts/eval-skills.sh b/scripts/eval-skills.sh new file mode 100755 index 00000000..c7b361cc --- /dev/null +++ b/scripts/eval-skills.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Score the skilld-maintained Skills with `claude plugin eval`. +# +# Each case runs twice: once with the Skills loaded, once without. The reported +# delta is the value the Skills add. Every run is a real Agent session on your +# own credential, so the suite costs money and is never part of `pnpm test`. +# +# Usage: scripts/eval-skills.sh [claude plugin eval flags] +# scripts/eval-skills.sh --case project-skill-real-paths +# scripts/eval-skills.sh --runs 1 --ablation none +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +stage="${TMPDIR:-/tmp}/skilld-eval-plugin" + +# The repository holds target/ and node_modules/, which overflow the eval's +# argument list. Stage only the plugin's own files. +rm -rf "$stage" +mkdir -p "$stage" +cp -r "$root/.claude-plugin" "$root/skills" "$root/evals" "$stage/" + +cd "$stage" +claude plugin eval . \ + --scaffold \ + --trust-plugin \ + --allow-tools Bash Write Edit \ + --no-publish \ + "$@" + +results="$stage/evals/results" +if [ -d "$results" ]; then + mkdir -p "$root/evals/results" + cp -r "$results/." "$root/evals/results/" + echo "Results copied to $root/evals/results" +fi From 4ba882ab1ea33e9869fad4124630c11b34390ad6 Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Tue, 22 Sep 2026 14:58:02 +1000 Subject: [PATCH 2/4] fix(harness): check only real path shapes and whole-word search commands --- .../harness/src/internal/output/project.ts | 24 ++++++++++++++--- .../harness/test/e2e/project-skill.test.ts | 26 +++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/harness/src/internal/output/project.ts b/packages/harness/src/internal/output/project.ts index 2ef698a2..768180a9 100644 --- a/packages/harness/src/internal/output/project.ts +++ b/packages/harness/src/internal/output/project.ts @@ -1,4 +1,5 @@ -const searchCommands = ['rg ', 'grep ', 'git grep '] +/** A search command the Agent can repeat, matched on whole words. */ +const searchCommand = /\b(?:git grep|grep|rg)\b/ /** Inline code spans, which is where a Skill writes its file pointers. */ const inlineCode = /`([^`\n]+)`/g @@ -21,6 +22,18 @@ function proseLines(markdown: string): ReadonlyArray { return markdown.replace(fencedBlock, '').split('\n') } +/** + * Extensions a real file pointer carries. A token ending elsewhere, such as + * the `.0` in a version, carries no path shape. + */ +const pathExtension = /\.(?:md|mdx|json|jsonc|ts|tsx|mts|cts|js|mjs|cjs|jsx|vue|svelte|txt|yml|yaml|toml|html|css|scss|rs|py|go|sh|lock|env|csv|xml)$/i + +/** A release number such as `20.19.0` or `v20` states a version, never a path. */ +const versionShape = /^v?\d+(?:\.\d+)*$/i + +/** A product name such as `Node.js` names a tool, never a file. */ +const productName = /^[A-Z][a-z][A-Za-z0-9_]*\.[A-Za-z0-9]+$/ + /** * A token counts as a file pointer only when it cannot be anything else. * A false pointer fails a correct Skill, so every rule here excludes rather than includes. @@ -37,8 +50,11 @@ function pathCandidate(token: string): string | null { const trimmed = token.replace(/^\.\//, '').replace(/\/+$/, '') if (trimmed.length === 0 || trimmed.startsWith('..')) return null - // A bare word such as `rg` or `pnpm` carries no path shape. - if (!trimmed.includes('/') && !/\.[a-z0-9]+$/i.test(trimmed)) + // A version, a product name, or a numbered reference such as `HTTP/2` is never a pointer. + if (versionShape.test(trimmed) || productName.test(trimmed) || /\/\d+$/.test(trimmed)) + return null + // A pointer carries a slash-qualified path or a known file extension. + if (!trimmed.includes('/') && !pathExtension.test(trimmed)) return null return trimmed } @@ -103,7 +119,7 @@ export function checkProjectSkill(input: ProjectSkillInput): ReadonlyArray match[1]!) const commands = [...fencedBlocks(input.markdown), ...inline].join('\n') - if (!searchCommands.some(command => commands.includes(command))) + if (!searchCommand.test(commands)) issues.push('SKILL.md must give the Agent at least one search command it can repeat.') return issues diff --git a/packages/harness/test/e2e/project-skill.test.ts b/packages/harness/test/e2e/project-skill.test.ts index 51b1bbb1..9bd3fb25 100644 --- a/packages/harness/test/e2e/project-skill.test.ts +++ b/packages/harness/test/e2e/project-skill.test.ts @@ -108,6 +108,32 @@ describe('project Skill navigation checks', () => { expect(result.error.issues).toContain('SKILL.md points at a path the project does not contain: src/server/routes.ts.') }) + it('accepts a version and a product name in inline code', async () => { + const body = conforming.replace( + 'The entry point is `src/index.ts`. The manifest is `package.json`.', + 'The entry point is `src/index.ts`. The manifest is `package.json`. The project runs on `Node.js` `20.19.0`.', + ) + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + + it('accepts a bare inline search command with no arguments', async () => { + const body = [ + '# Example project', + '', + 'The entry point is `src/index.ts`.', + '', + 'Search the source with `rg`.', + '', + ].join('\n') + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + it('refuses a project Skill that gives the Agent no search command', async () => { const body = ['# Example project', '', 'The entry point is `src/index.ts`.', ''].join('\n') From b7855f118badad47535c76d4f3367eafb9ebf5bd Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Tue, 22 Sep 2026 15:26:42 +1000 Subject: [PATCH 3/4] fix(harness): accept lowercase product names and files a Skill creates --- .../harness/src/internal/output/project.ts | 18 ++++++---- .../harness/test/e2e/project-skill.test.ts | 36 ++++++++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/harness/src/internal/output/project.ts b/packages/harness/src/internal/output/project.ts index 768180a9..d2c282e9 100644 --- a/packages/harness/src/internal/output/project.ts +++ b/packages/harness/src/internal/output/project.ts @@ -10,9 +10,12 @@ const fencedBlock = /```[^\n]*\n[\s\S]*?```/g /** * A Skill may report a file the project lacks, such as a missing entry point. * A line that denies a path states a fact about the project, not a pointer. + * A line that tells the Agent to create a file gives a task, not a pointer. */ const denial = /\b(no|not|none|never|absent|missing|lacks|without|instead|rather|empty|undeclared|unavailable|nothing)\b|\bdoes ?n[o']t\b/i +const creation = /\b(?:copy|create|write|generate)\b/i + function fencedBlocks(markdown: string): ReadonlyArray { return markdown.match(fencedBlock) ?? [] } @@ -31,8 +34,8 @@ const pathExtension = /\.(?:md|mdx|json|jsonc|ts|tsx|mts|cts|js|mjs|cjs|jsx|vue| /** A release number such as `20.19.0` or `v20` states a version, never a path. */ const versionShape = /^v?\d+(?:\.\d+)*$/i -/** A product name such as `Node.js` names a tool, never a file. */ -const productName = /^[A-Z][a-z][A-Za-z0-9_]*\.[A-Za-z0-9]+$/ +/** A product name such as `Node.js` or `node.js` names a tool, never a file. */ +const productName = /^[A-Za-z][A-Za-z0-9_]*\.[A-Za-z0-9]+$/ /** * A token counts as a file pointer only when it cannot be anything else. @@ -74,8 +77,11 @@ function resolves(candidate: string, known: ReadonlySet): boolean { const prefix = literalPrefix(candidate) if (prefix.length === 0) return true - // A directory pointer resolves when the project holds a file under it. for (const path of known) { + // A known template such as `.env.example` implies the file it is copied to. + if (path.startsWith(`${candidate}.`)) + return true + // A directory pointer resolves when the project holds a file under it. if (path === prefix || path.startsWith(`${prefix}/`)) return true } @@ -96,8 +102,8 @@ export interface ProjectSkillInput { * search the Agent can repeat. The Agent cannot verify its own pointers. * * The Harness hides generated directories from the Agent, so a pointer at one is - * invention like any other. A line that states a path is missing passes, because - * reporting an absent file is a fact rather than a pointer. + * invention like any other. A line that states a path is missing, or that tells + * the Agent to create one, passes: it reports a fact or gives a task, not a pointer. */ export function checkProjectSkill(input: ProjectSkillInput): ReadonlyArray { const issues: string[] = [] @@ -105,7 +111,7 @@ export function checkProjectSkill(input: ProjectSkillInput): ReadonlyArray() for (const line of proseLines(input.markdown)) { - if (denial.test(line)) + if (denial.test(line) || creation.test(line)) continue for (const match of line.matchAll(inlineCode)) { const candidate = pathCandidate(match[1]!.trim()) diff --git a/packages/harness/test/e2e/project-skill.test.ts b/packages/harness/test/e2e/project-skill.test.ts index 9bd3fb25..72077a18 100644 --- a/packages/harness/test/e2e/project-skill.test.ts +++ b/packages/harness/test/e2e/project-skill.test.ts @@ -111,7 +111,7 @@ describe('project Skill navigation checks', () => { it('accepts a version and a product name in inline code', async () => { const body = conforming.replace( 'The entry point is `src/index.ts`. The manifest is `package.json`.', - 'The entry point is `src/index.ts`. The manifest is `package.json`. The project runs on `Node.js` `20.19.0`.', + 'The entry point is `src/index.ts`. The manifest is `package.json`. The project runs on `Node.js` `20.19.0`. The runtime is `node.js` and `vue.js`.', ) const { result } = await runProjectSkill(projectSkill(body)) @@ -119,6 +119,40 @@ describe('project Skill navigation checks', () => { expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) }) + it('accepts a file a line tells the Agent to copy from a template', async () => { + const projectDir = await makeProject() + await writeFile(join(projectDir, '.env.example'), 'SECRET=\n') + const destinationRoot = await mkdtemp(join(tmpdir(), 'skilld-project-nav-out-')) + const body = [ + '# Example project', + '', + 'The entry point is `src/index.ts`.', + '', + 'Copy `.env.example` to `.env` for local secrets.', + '', + '```sh', + 'rg -n "export " src', + '```', + '', + ].join('\n') + const { harness } = createFakeHarness({ + async onPrompt({ sandbox, workDir }) { + await sandbox.writeTextFile({ + path: join(workDir, 'skilld-output/example-project/SKILL.md'), + content: projectSkill(body), + }) + }, + }) + + const result = await createSkillHarness({ harness, sandbox: createFakeSandboxProvider() }).run({ + _tag: 'ProjectSkill', + projectDir, + destination: { rootDir: destinationRoot, name: 'example-project' }, + }) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + it('accepts a bare inline search command with no arguments', async () => { const body = [ '# Example project', From fb6d30c8537f761955e1e9e34d6bbb40d635a257 Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Tue, 22 Sep 2026 15:39:53 +1000 Subject: [PATCH 4/4] fix(harness): carry denial state across wrapped prose lines --- .../harness/src/internal/output/project.ts | 31 +++++++++++++------ .../harness/test/e2e/project-skill.test.ts | 11 +++++++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/harness/src/internal/output/project.ts b/packages/harness/src/internal/output/project.ts index d2c282e9..5b01a2ef 100644 --- a/packages/harness/src/internal/output/project.ts +++ b/packages/harness/src/internal/output/project.ts @@ -9,8 +9,10 @@ const fencedBlock = /```[^\n]*\n[\s\S]*?```/g /** * A Skill may report a file the project lacks, such as a missing entry point. - * A line that denies a path states a fact about the project, not a pointer. - * A line that tells the Agent to create a file gives a task, not a pointer. + * A sentence that denies a path states a fact about the project, not a pointer. + * A sentence that tells the Agent to create a file gives a task, not a pointer. + * The check reads whole paragraphs, so a hard-wrapped denial still covers the + * code spans in its own sentence. */ const denial = /\b(no|not|none|never|absent|missing|lacks|without|instead|rather|empty|undeclared|unavailable|nothing)\b|\bdoes ?n[o']t\b/i @@ -20,9 +22,17 @@ function fencedBlocks(markdown: string): ReadonlyArray { return markdown.match(fencedBlock) ?? [] } -/** Prose lines, with fenced commands removed so a command never reads as a pointer. */ -function proseLines(markdown: string): ReadonlyArray { - return markdown.replace(fencedBlock, '').split('\n') +/** + * Prose paragraphs, with fenced commands removed so a command never reads as a + * pointer. A soft-wrapped sentence keeps its words together: the lines of a + * paragraph join before any check runs. + */ +function proseParagraphs(markdown: string): ReadonlyArray { + return markdown + .replace(fencedBlock, '') + .split(/\n[ \t]*\n/) + .map(paragraph => paragraph.replace(/\n/g, ' ').trim()) + .filter(paragraph => paragraph.length > 0) } /** @@ -102,18 +112,19 @@ export interface ProjectSkillInput { * search the Agent can repeat. The Agent cannot verify its own pointers. * * The Harness hides generated directories from the Agent, so a pointer at one is - * invention like any other. A line that states a path is missing, or that tells - * the Agent to create one, passes: it reports a fact or gives a task, not a pointer. + * invention like any other. A sentence that states a path is missing, or that + * tells the Agent to create one, passes: it reports a fact or gives a task, not + * a pointer. The sentence may wrap across prose lines within its paragraph. */ export function checkProjectSkill(input: ProjectSkillInput): ReadonlyArray { const issues: string[] = [] const known = new Set([...input.projectPaths, ...input.outputPaths]) const unknown = new Set() - for (const line of proseLines(input.markdown)) { - if (denial.test(line) || creation.test(line)) + for (const paragraph of proseParagraphs(input.markdown)) { + if (denial.test(paragraph) || creation.test(paragraph)) continue - for (const match of line.matchAll(inlineCode)) { + for (const match of paragraph.matchAll(inlineCode)) { const candidate = pathCandidate(match[1]!.trim()) if (candidate !== null && !resolves(candidate, known)) unknown.add(candidate) diff --git a/packages/harness/test/e2e/project-skill.test.ts b/packages/harness/test/e2e/project-skill.test.ts index 72077a18..36fa997e 100644 --- a/packages/harness/test/e2e/project-skill.test.ts +++ b/packages/harness/test/e2e/project-skill.test.ts @@ -84,6 +84,17 @@ describe('project Skill navigation checks', () => { expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) }) + it('accepts a denial sentence wrapped across two prose lines', async () => { + const body = conforming.replace( + 'The entry point is `src/index.ts`. The manifest is `package.json`.', + 'The project lacks\n`src/store.ts` so the import must be added first.', + ) + + const { result } = await runProjectSkill(projectSkill(body)) + + expect(result).toMatchObject({ _tag: 'Ok', value: { _tag: 'GeneratedSkill' } }) + }) + it('accepts a generated path a line puts off limits', async () => { const body = conforming.replace( 'The entry point is `src/index.ts`.',