feat(harness): fail a project Skill that points at a path it invented

.claude-plugin/plugin.json

@@ -0,0 +1,8 @@
+{
+  "name": "skilld",
+  "description": "The skilld-maintained Skills for Skill generation, review, and skilld CLI use.",
+  "version": "3.1.0",
+  "experimental": {
+    "evals": "evals"
+  }
+}

.gitignore

@@ -29,3 +29,4 @@ coverage
 .artifacts
 
 .guides-out
+evals/results

CLAUDE.md

@@ -22,6 +22,15 @@ cargo test --workspace
 cargo clippy --workspace --all-targets -- -D warnings
 ```
 
+Score the skilld-maintained Skills against scaffolded projects:
+
+```sh
+./scripts/eval-skills.sh
+```
+
+Every case runs a real Agent session and costs money. Read `evals/README.md`
+before changing a case or a grader.
+
 ## Product boundary
 
 The native `skilld` CLI searches, runs, installs, lists, views, removes, updates, and verifies Skills.

CONTEXT.md

@@ -18,6 +18,12 @@ Direct runs remain user reviewed.
 
 Harness runs apply deterministic checks before atomic promotion.
 
+A project Skill must point at prepared project files.
+
+The Harness rejects a pointer the prepared source does not contain.
+
+`evals` scores the same Skills on the direct path, where no check runs.
+
 ## Trust
 
 - `verified` records an Artifact with a valid attestation.

evals/README.md

@@ -0,0 +1,53 @@
+# skilld-maintained Skill evals
+
+These cases score the Skills in `skills/` against scaffolded projects.
+
+Direct Skill runs have no enforcement. An Agent reads the Skill and writes files.
+These cases measure what the Agent actually produces.
+
+## Run them
+
+```sh
+./scripts/eval-skills.sh
+./scripts/eval-skills.sh --case project-skill-rust --runs 1
+```
+
+Each run is a real Agent session on your own credential. The suite costs money,
+so it never runs in `pnpm test` or in CI on a pull request.
+
+The script stages `skills/`, `evals/`, and `.claude-plugin/` into a temporary
+directory before it runs. The repository holds `target/` and `node_modules/`,
+which overflow the eval's argument list.
+
+## Read the score
+
+Every case runs twice: once with the Skills loaded, once without. The delta is
+the value the Skills add. A case that scores the same in both arms measures the
+model, not the Skill.
+
+Recorded on 2026-09-22, 2 runs per arm, Claude Code 2.1.278:
+
+| Case | With | Without | Δ |
+| --- | --- | --- | --- |
+| `package-not-project` | 1.00 | 0.67 | +0.33 |
+| `project-skill-real-paths` | 1.00 | 0.75 | +0.25 |
+| `project-skill-rust` | 1.00 | 0.75 | +0.25 |
+
+## What each case holds
+
+- `project-skill-real-paths`: a TypeScript project with a `dist/` decoy. Checks
+  that the Skill names real paths and gives a search the Agent can repeat.
+- `project-skill-rust`: a Cargo project with no `package.json`. Checks that the
+  Skill reads the project's own manifest and finds its declared binary.
+- `package-not-project`: a published package. Checks that a request for consumer
+  instructions routes to `generate-package-skill`.
+
+## Write a case
+
+A grader reads the transcript, not the files the Agent wrote. Ask the prompt for
+a closing summary that states what a grader needs to see.
+
+An `llm` grader judges shape and intent. A `regex` or `file_exists` grader
+settles a fact. Deterministic rules about SKILL.md content belong in the Harness
+instead, where `packages/harness/src/internal/output/project.ts` fails a Skill
+before promotion.

evals/package-not-project/case.yaml

@@ -0,0 +1,17 @@
+schema_version: '1.0'
+name: package-not-project
+description: A Skill for a package's consumers uses the package Skill path, not the project one.
+tags: [project-skill, routing]
+runs: 2
+context:
+  scaffold_script: ./scaffold.sh
+execution:
+  max_turns: 30
+  allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit]
+  prompt: |
+    I maintain the `tinyfmt` package published on npm. Draft an Agent Skill that
+    teaches an Agent to use `tinyfmt` in someone else's codebase. Write it to
+    `skills/tinyfmt/SKILL.md`.
+
+    End with a short summary that says which skilld-maintained Skill you followed
+    and why.

evals/package-not-project/graders/criteria.md

@@ -0,0 +1,20 @@
+---
+type: llm
+weight: 2
+focus: last_message
+---
+
+The request is for a Skill that teaches consumers to use the `tinyfmt` package,
+not for a Skill that navigates this workspace.
+
+A passing run:
+
+- Follows `generate-package-skill`, and says so.
+- Writes consumer-facing instructions: the public export `formatMoney`, its
+  arguments, and how to import it.
+- Does not turn the result into a project navigation Skill. A layout table of
+  this repository, or search commands for changing this repository, is the wrong
+  shape here.
+
+It fails if it follows `generate-project-skill`, or if the Skill it writes reads
+as internal navigation for this workspace rather than usage for a consumer.

evals/package-not-project/graders/skill-fired.md

@@ -0,0 +1,6 @@
+---
+type: tool_used
+tool: Skill
+min: 1
+arm: with-only
+---

evals/package-not-project/graders/skill-written.md

@@ -0,0 +1,5 @@
+---
+type: file_exists
+path: skills/tinyfmt/SKILL.md
+exists: true
+---

evals/package-not-project/scaffold.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+mkdir -p src test
+cat > package.json <<'JSON'
+{
+  "name": "tinyfmt",
+  "description": "Format a number as a currency string.",
+  "type": "module",
+  "version": "1.4.0",
+  "exports": "./src/index.js",
+  "types": "./src/index.d.ts"
+}
+JSON
+printf '# tinyfmt\n\nFormat a number as a currency string.\n\n## Usage\n\n```js\nimport { formatMoney } from "tinyfmt"\n```\n' > README.md
+printf 'export const formatMoney = (cents, currency = "USD") => new Intl.NumberFormat("en-US", { style: "currency", currency }).format(cents / 100)\n' > src/index.js
+printf 'export declare const formatMoney: (cents: number, currency?: string) => string\n' > src/index.d.ts
+printf 'import { formatMoney } from "../src/index.js"\nconsole.log(formatMoney(1234))\n' > test/format.test.js

evals/project-skill-real-paths/case.yaml

@@ -0,0 +1,16 @@
+schema_version: '1.0'
+name: project-skill-real-paths
+description: A project Skill drafted for this workspace points only at files the project contains.
+tags: [project-skill]
+runs: 2
+context:
+  scaffold_script: ./scaffold.sh
+execution:
+  max_turns: 30
+  allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit]
+  prompt: |
+    Draft an Agent Skill for this project so a future session can navigate it
+    without reading every file first. Write it to `skills/invoicer/SKILL.md`.
+
+    End with a short summary that quotes one search command from the Skill and
+    lists every project path the Skill names.

evals/project-skill-real-paths/graders/criteria.md

@@ -0,0 +1,27 @@
+---
+type: llm
+weight: 2
+focus: last_message
+---
+
+The run drafted a project Skill for the `invoicer` workspace.
+
+A passing run:
+
+- Names only paths the project contains. These exist: `package.json`,
+  `README.md`, `src/index.ts`, `src/core/render.ts`, `src/core/send.ts`,
+  `src/commands/send.ts`, `docs/ledger.md`, `dist/index.js`, `dist/send.js`.
+  Any other source path is invented.
+- Names the entry point the manifest declares, `src/index.ts`.
+- Gives at least one search command the Agent can repeat, such as an `rg`
+  invocation scoped to a real directory.
+- Treats `dist/` as generated output. Naming it as a directory to leave alone
+  passes. Pointing at it as a place to read or edit source fails.
+- Uses project-relative paths, not absolute ones.
+
+Naming a path the project lacks passes when the Skill says it is missing. A
+report such as "there is no `src/lib.rs`" is a fact about the project.
+Presenting a path that does not exist as a real file is what fails.
+
+It fails if it names a path the project does not contain, sends the Agent into
+`dist/` to read source, or gives no repeatable search command.

evals/project-skill-real-paths/graders/search-command.md

@@ -0,0 +1,7 @@
+---
+type: regex
+target: last_message
+pattern: "(rg|grep) "
+match: contains
+weight: 1
+---

evals/project-skill-real-paths/graders/skill-fired.md

@@ -0,0 +1,6 @@
+---
+type: tool_used
+tool: Skill
+min: 1
+arm: with-only
+---

evals/project-skill-real-paths/graders/skill-written.md

@@ -0,0 +1,5 @@
+---
+type: file_exists
+path: skills/invoicer/SKILL.md
+exists: true
+---

evals/project-skill-real-paths/scaffold.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+mkdir -p src/commands src/core docs dist
+cat > package.json <<'JSON'
+{
+  "name": "invoicer",
+  "description": "Render and send invoices from a ledger file.",
+  "type": "module",
+  "main": "./src/index.ts",
+  "scripts": {
+    "test": "vitest run",
+    "lint": "eslint ."
+  }
+}
+JSON
+printf '# invoicer\n\nRender and send invoices from a ledger file.\n' > README.md
+printf 'export { renderInvoice } from "./core/render.ts"\nexport { sendInvoice } from "./core/send.ts"\n' > src/index.ts
+printf 'export const renderInvoice = (ledger: string): string => ledger.trim()\n' > src/core/render.ts
+printf 'export const sendInvoice = async (body: string): Promise<void> => { void body }\n' > src/core/send.ts
+printf 'export const runSend = async (): Promise<void> => {}\n' > src/commands/send.ts
+printf '# Ledger format\n\nOne invoice per line.\n' > docs/ledger.md
+printf 'export const renderInvoice=(l)=>l.trim();\n' > dist/index.js
+printf 'export const sendInvoice=async(b)=>{};\n' > dist/send.js

evals/project-skill-rust/case.yaml

@@ -0,0 +1,17 @@
+schema_version: '1.0'
+name: project-skill-rust
+description: A project Skill reads the project's own manifest, not a JavaScript one, and finds its real entry point.
+tags: [project-skill]
+runs: 2
+context:
+  scaffold_script: ./scaffold.sh
+execution:
+  max_turns: 30
+  allowed_tools: [Read, Glob, Grep, Skill, Bash, Write, Edit]
+  prompt: |
+    Draft an Agent Skill for this project so a future session can navigate it
+    without reading every file first. Write it to `skills/ledgerd/SKILL.md`.
+
+    End with a short summary that quotes one search command from the Skill and
+    lists every project path the Skill names and
+    the entry point it found.

evals/project-skill-rust/graders/criteria.md

@@ -0,0 +1,26 @@
+---
+type: llm
+weight: 2
+focus: last_message
+---
+
+The run drafted a project Skill for the `ledgerd` Rust workspace.
+
+A passing run:
+
+- Reads `Cargo.toml` for the project name and entry point. It never claims a
+  `package.json`, which this project does not have.
+- Names `src/main.rs` as the binary entry point, as `[[bin]]` declares.
+- Names only paths the project contains: `Cargo.toml`, `README.md`,
+  `src/main.rs`, `src/store/mod.rs`, `src/store/journal.rs`, `tests/journal.rs`,
+  `target/debug/ledgerd`. Any other source path is invented.
+- Gives at least one search command the Agent can repeat, scoped to a real
+  directory such as `src`.
+- Treats `target/` as generated output rather than source to read.
+
+Naming a path the project lacks passes when the Skill says it is missing. A
+report such as "there is no `src/lib.rs`" is a fact about the project.
+Presenting a path that does not exist as a real file is what fails.
+
+It fails if it invents a path, claims a manifest the project does not have,
+names the wrong entry point, or gives no repeatable search command.

evals/project-skill-rust/graders/search-command.md

@@ -0,0 +1,7 @@
+---
+type: regex
+target: last_message
+pattern: "(rg|grep) "
+match: contains
+weight: 1
+---

evals/project-skill-rust/graders/skill-fired.md

@@ -0,0 +1,6 @@
+---
+type: tool_used
+tool: Skill
+min: 1
+arm: with-only
+---

evals/project-skill-rust/graders/skill-written.md

@@ -0,0 +1,5 @@
+---
+type: file_exists
+path: skills/ledgerd/SKILL.md
+exists: true
+---

evals/project-skill-rust/scaffold.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+mkdir -p src/store tests target/debug
+cat > Cargo.toml <<'TOML'
+[package]
+name = "ledgerd"
+version = "0.2.0"
+edition = "2021"
+description = "Append-only ledger daemon."
+
+[[bin]]
+name = "ledgerd"
+path = "src/main.rs"
+TOML
+printf '# ledgerd\n\nAppend-only ledger daemon.\n' > README.md
+printf 'mod store;\n\nfn main() {\n    store::open();\n}\n' > src/main.rs
+printf 'pub mod journal;\n\npub fn open() {}\n' > src/store/mod.rs
+printf 'pub fn append(entry: &str) -> usize {\n    entry.len()\n}\n' > src/store/journal.rs
+printf '#[test]\nfn appends_an_entry() {\n    assert_eq!(ledgerd::store::journal::append("a"), 1);\n}\n' > tests/journal.rs
+printf 'binary\n' > target/debug/ledgerd

packages/harness/src/harness.ts

@@ -237,7 +237,13 @@ export function createSkillHarness(options: CreateSkillHarnessOptions): SkillHar
         if (prepared.value.skillName === 'review-skill')
           return validateSkillReview(collected.value)
 
-        const validated = validateGeneratedSkill(prepared.value.outputName, collected.value)
+        const validated = validateGeneratedSkill(
+          prepared.value.outputName,
+          collected.value,
+          prepared.value.skillName === 'generate-project-skill'
+            ? prepared.value.source.files.map(file => file.path)
+            : undefined,
+        )
         if (validated._tag === 'Err')
           return validated
         if (!prepared.value.destination)