diff --git a/src/hooks/codex/wiki-worker.ts b/src/hooks/codex/wiki-worker.ts index 3175c73c..ff274587 100644 --- a/src/hooks/codex/wiki-worker.ts +++ b/src/hooks/codex/wiki-worker.ts @@ -13,7 +13,7 @@ import { buildTrailingPromptInvocation } from "../wiki-worker-spawn.js"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; -import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; @@ -135,19 +135,27 @@ function formatExecFailure(error: any): string { async function main(): Promise { try { - // 1. Fetch session events from sessions table + // 1. Fetch session events from the sessions table — BOUNDED to the newest + // WIKI_FALLBACK_MAX_ROWS rows (reversed to chronological) plus the true + // total. The old unbounded `ORDER BY ASC` materialized the whole fat + // `message` column (tens of MB, ~30s cold on a mega-session) even though + // only the newest un-summarized rows are consumed. `count(*)` reads no fat + // column, so `total` is cheap and keeps the offset math + stamped offset right. wlog("fetching session events"); - const rows = await query( - `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + - `WHERE path LIKE E'${esc(`/sessions/%${cfg.sessionId}%`)}' ORDER BY creation_date ASC` - ); - - if (rows.length === 0) { + const likePat = esc(`/sessions/%${cfg.sessionId}%`); + const cntRows = await query(`SELECT count(*) AS n FROM "${cfg.sessionsTable}" WHERE path LIKE E'${likePat}'`); + const total = Number(cntRows[0]?.["n"] ?? 0); + if (total === 0) { wlog("no session events found — exiting"); return; } + const fetched = await query( + `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + + `WHERE path LIKE E'${likePat}' ORDER BY creation_date DESC LIMIT ${WIKI_FALLBACK_MAX_ROWS}` + ); + const rows = fetched.reverse(); - const jsonlLines = rows.length; + const jsonlLines = total; const pathRows = await query( `SELECT DISTINCT path FROM "${cfg.sessionsTable}" ` + @@ -201,7 +209,7 @@ async function main(): Promise { // full session on every run is what drove the ENOBUFS / 120s-timeout // failures on long (4000+ event) sessions — a stuck offset meant every run // re-summarized everything from scratch. - const newRows = prevOffset > 0 ? rows.slice(prevOffset) : rows; + const newRows = newRowsFromWindow(rows, total, prevOffset); if (prevOffset > 0 && newRows.length === 0) { wlog(`no new events since last summary (offset=${prevOffset}, total=${jsonlLines}) — skipping`); return; diff --git a/src/hooks/cursor/wiki-worker.ts b/src/hooks/cursor/wiki-worker.ts index 5a5a7a56..c7a03c27 100644 --- a/src/hooks/cursor/wiki-worker.ts +++ b/src/hooks/cursor/wiki-worker.ts @@ -20,7 +20,7 @@ import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { readSessionEventCache } from "../session-event-cache.js"; import { buildSessionPath } from "../../utils/session-path.js"; -import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; @@ -126,12 +126,26 @@ async function main(): Promise { // mega-sessions). Falls back to the DB whenever the cache is absent // (session resumed on another machine), empty, or — once the offset is // known — shorter than the offset already summarized. - const dbFetch = () => query( - `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + - `WHERE path LIKE E'${esc(`/sessions/%${cfg.sessionId}%`)}' ORDER BY creation_date ASC` - ); + // Bounded DB fallback: the NEWEST WIKI_FALLBACK_MAX_ROWS rows (reversed to + // chronological) plus the true `total`. The old unbounded `ORDER BY ASC` + // materialized the whole fat `message` column (tens of MB, ~30s cold on a + // mega-session) even though only the newest un-summarized rows are consumed. + // `count(*)` reads no fat column, so `total` is cheap and lets the offset + // math (`newRowsFromWindow`) and the stamped offset stay correct. + const dbFetch = async (): Promise<{ rows: Record[]; total: number }> => { + const like = esc(`/sessions/%${cfg.sessionId}%`); + const cnt = await query(`SELECT count(*) AS n FROM "${cfg.sessionsTable}" WHERE path LIKE E'${like}'`); + const total = Number(cnt[0]?.["n"] ?? 0); + if (total === 0) return { rows: [], total: 0 }; + const r = await query( + `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + + `WHERE path LIKE E'${like}' ORDER BY creation_date DESC LIMIT ${WIKI_FALLBACK_MAX_ROWS}` + ); + return { rows: r.reverse(), total }; + }; let usedLocalCache = false; + let dbTotal = 0; // true row count when the bounded DB path was used (else 0) let rows: Record[]; const cachedLines = readSessionEventCache(cfg.sessionId); if (cachedLines && cachedLines.length > 0) { @@ -140,7 +154,9 @@ async function main(): Promise { wlog(`loaded ${rows.length} events from local cache`); } else { wlog("fetching session events"); - rows = await dbFetch(); + const f = await dbFetch(); + rows = f.rows; + dbTotal = f.total; } if (rows.length === 0) { @@ -148,7 +164,9 @@ async function main(): Promise { return; } - let jsonlLines = rows.length; + // The offset high-water (stamped into the summary) must be the TRUE total, not the + // bounded window length — else the next run's offset regresses and re-summarizes. + let jsonlLines = usedLocalCache ? rows.length : dbTotal; // Derive the server path locally when using the cache (avoids a second // self-session `SELECT DISTINCT path` scan); the DB branch keeps its lookup. @@ -213,15 +231,20 @@ async function main(): Promise { // session from the DB so no genuinely-new rows get sliced to nothing. if (usedLocalCache && rows.length < prevOffset) { wlog(`local cache (${rows.length}) < summarized offset (${prevOffset}) — refetching from DB`); - rows = await dbFetch(); - jsonlLines = rows.length; + const f = await dbFetch(); + rows = f.rows; + dbTotal = f.total; + jsonlLines = dbTotal; + usedLocalCache = false; } // Feed the agent only the rows added since the last summary. Reprocessing // the full session on every run is what drives ENOBUFS / 120s-timeout // failures on long (4000+ event) sessions — a stuck offset re-summarizes // everything from scratch. - const newRows = prevOffset > 0 ? rows.slice(prevOffset) : rows; + const newRows = usedLocalCache + ? (prevOffset > 0 ? rows.slice(prevOffset) : rows) + : newRowsFromWindow(rows, dbTotal, prevOffset); if (prevOffset > 0 && newRows.length === 0) { wlog(`no new events since last summary (offset=${prevOffset}, total=${jsonlLines}) — skipping`); return; diff --git a/src/hooks/hermes/wiki-worker.ts b/src/hooks/hermes/wiki-worker.ts index d1d04d5c..fcd293d6 100644 --- a/src/hooks/hermes/wiki-worker.ts +++ b/src/hooks/hermes/wiki-worker.ts @@ -19,7 +19,7 @@ import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; import { readSessionEventCache } from "../session-event-cache.js"; import { buildSessionPath } from "../../utils/session-path.js"; -import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; @@ -126,12 +126,26 @@ async function main(): Promise { // mega-sessions). Falls back to the DB whenever the cache is absent // (session resumed on another machine), empty, or — once the offset is // known — shorter than the offset already summarized. - const dbFetch = () => query( - `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + - `WHERE path LIKE E'${esc(`/sessions/%${cfg.sessionId}%`)}' ORDER BY creation_date ASC` - ); + // Bounded DB fallback: the NEWEST WIKI_FALLBACK_MAX_ROWS rows (reversed to + // chronological) plus the true `total`. The old unbounded `ORDER BY ASC` + // materialized the whole fat `message` column (tens of MB, ~30s cold on a + // mega-session) even though only the newest un-summarized rows are consumed. + // `count(*)` reads no fat column, so `total` is cheap and lets the offset + // math (`newRowsFromWindow`) and the stamped offset stay correct. + const dbFetch = async (): Promise<{ rows: Record[]; total: number }> => { + const like = esc(`/sessions/%${cfg.sessionId}%`); + const cnt = await query(`SELECT count(*) AS n FROM "${cfg.sessionsTable}" WHERE path LIKE E'${like}'`); + const total = Number(cnt[0]?.["n"] ?? 0); + if (total === 0) return { rows: [], total: 0 }; + const r = await query( + `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + + `WHERE path LIKE E'${like}' ORDER BY creation_date DESC LIMIT ${WIKI_FALLBACK_MAX_ROWS}` + ); + return { rows: r.reverse(), total }; + }; let usedLocalCache = false; + let dbTotal = 0; // true row count when the bounded DB path was used (else 0) let rows: Record[]; const cachedLines = readSessionEventCache(cfg.sessionId); if (cachedLines && cachedLines.length > 0) { @@ -140,7 +154,9 @@ async function main(): Promise { wlog(`loaded ${rows.length} events from local cache`); } else { wlog("fetching session events"); - rows = await dbFetch(); + const f = await dbFetch(); + rows = f.rows; + dbTotal = f.total; } if (rows.length === 0) { @@ -148,7 +164,9 @@ async function main(): Promise { return; } - let jsonlLines = rows.length; + // The offset high-water (stamped into the summary) must be the TRUE total, not the + // bounded window length — else the next run's offset regresses and re-summarizes. + let jsonlLines = usedLocalCache ? rows.length : dbTotal; // Derive the server path locally when using the cache (avoids a second // self-session `SELECT DISTINCT path` scan); the DB branch keeps its lookup. @@ -213,15 +231,22 @@ async function main(): Promise { // session from the DB so no genuinely-new rows get sliced to nothing. if (usedLocalCache && rows.length < prevOffset) { wlog(`local cache (${rows.length}) < summarized offset (${prevOffset}) — refetching from DB`); - rows = await dbFetch(); - jsonlLines = rows.length; + const f = await dbFetch(); + rows = f.rows; + dbTotal = f.total; + jsonlLines = dbTotal; + usedLocalCache = false; } // Feed the agent only the rows added since the last summary. Reprocessing // the full session on every run is what drives ENOBUFS / 120s-timeout // failures on long (4000+ event) sessions — a stuck offset re-summarizes - // everything from scratch. - const newRows = prevOffset > 0 ? rows.slice(prevOffset) : rows; + // everything from scratch. The local-cache path holds the FULL session, so a + // plain slice is exact; the bounded DB window maps the full-history offset + // through newRowsFromWindow. + const newRows = usedLocalCache + ? (prevOffset > 0 ? rows.slice(prevOffset) : rows) + : newRowsFromWindow(rows, dbTotal, prevOffset); if (prevOffset > 0 && newRows.length === 0) { wlog(`no new events since last summary (offset=${prevOffset}, total=${jsonlLines}) — skipping`); return; diff --git a/src/hooks/pi/wiki-worker.ts b/src/hooks/pi/wiki-worker.ts index 23138170..8b03fb66 100644 --- a/src/hooks/pi/wiki-worker.ts +++ b/src/hooks/pi/wiki-worker.ts @@ -22,7 +22,7 @@ import { buildTrailingPromptInvocation } from "../wiki-worker-spawn.js"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { finalizeSummary, releaseLock, readState } from "../summary-state.js"; -import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; +import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "../wiki-offset.js"; import { redactSecrets } from "../shared/redact.js"; import { uploadSummary } from "../upload-summary.js"; import { log as _log } from "../../utils/debug.js"; @@ -120,19 +120,27 @@ function cleanup(): void { async function main(): Promise { try { - // 1. Fetch session events from sessions table + // 1. Fetch session events from the sessions table — BOUNDED to the newest + // WIKI_FALLBACK_MAX_ROWS rows (reversed to chronological) plus the true + // total. The old unbounded `ORDER BY ASC` materialized the whole fat + // `message` column (tens of MB, ~30s cold on a mega-session) even though + // only the newest un-summarized rows are consumed. `count(*)` reads no fat + // column, so `total` is cheap and keeps the offset math + stamped offset right. wlog("fetching session events"); - const rows = await query( - `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + - `WHERE path LIKE E'${esc(`/sessions/%${cfg.sessionId}%`)}' ORDER BY creation_date ASC` - ); - - if (rows.length === 0) { + const likePat = esc(`/sessions/%${cfg.sessionId}%`); + const cntRows = await query(`SELECT count(*) AS n FROM "${cfg.sessionsTable}" WHERE path LIKE E'${likePat}'`); + const total = Number(cntRows[0]?.["n"] ?? 0); + if (total === 0) { wlog("no session events found — exiting"); return; } + const fetched = await query( + `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + + `WHERE path LIKE E'${likePat}' ORDER BY creation_date DESC LIMIT ${WIKI_FALLBACK_MAX_ROWS}` + ); + const rows = fetched.reverse(); - const jsonlLines = rows.length; + const jsonlLines = total; const pathRows = await query( `SELECT DISTINCT path FROM "${cfg.sessionsTable}" ` + @@ -186,7 +194,7 @@ async function main(): Promise { // the full session on every run is what drives ENOBUFS / 120s-timeout // failures on long (4000+ event) sessions — a stuck offset re-summarizes // everything from scratch. - const newRows = prevOffset > 0 ? rows.slice(prevOffset) : rows; + const newRows = newRowsFromWindow(rows, total, prevOffset); if (prevOffset > 0 && newRows.length === 0) { wlog(`no new events since last summary (offset=${prevOffset}, total=${jsonlLines}) — skipping`); return; diff --git a/src/hooks/wiki-offset.ts b/src/hooks/wiki-offset.ts index ed140e2d..633c7b5d 100644 --- a/src/hooks/wiki-offset.ts +++ b/src/hooks/wiki-offset.ts @@ -18,6 +18,39 @@ /** Max bytes of session JSONL fed to the summarizer in one run. */ export const WIKI_JSONL_MAX_BYTES = 4 * 1024 * 1024; +/** + * Max session rows the DB fallback fetches, newest-first. The old fallback did an + * unbounded `ORDER BY creation_date ASC` with no limit, which materializes the WHOLE fat + * `message` column — tens of MB, ~30s cold on a mega-session — even though the summarizer + * only ever consumes the newest un-summarized rows (`capLinesByBytes` discards the rest). + * This caps the fetch to a superset of what any one run can use. + */ +export const WIKI_FALLBACK_MAX_ROWS = 2000; + +/** + * Select the un-summarized "new" rows from a BOUNDED newest-N window. + * + * The workers fetch the whole session ASC and take `rows.slice(prevOffset)`, where + * `prevOffset` is a count over the FULL history. When the fetch is instead bounded to the + * newest `window.length` rows (of a `total`-row session), that full-history index no longer + * addresses the window, so the plain slice is wrong. This maps it correctly: + * + * newCount = max(0, total - prevOffset) // rows added since the last summary + * → the LAST `newCount` rows of the window are the un-summarized ones. + * + * When the window doesn't reach back to `prevOffset` (the session grew by more than the + * window since the last summary), `newCount >= window.length` and the whole window is + * returned — the older new rows fell outside the fetch, exactly the ones `capLinesByBytes` + * would drop anyway (it keeps the newest). So bounding never changes what a run summarizes + * beyond what the byte cap already does. `prevOffset <= 0` returns the whole window. + */ +export function newRowsFromWindow(window: readonly T[], total: number, prevOffset: number): T[] { + if (prevOffset <= 0) return window.slice(); + const newCount = Math.max(0, total - prevOffset); + if (newCount >= window.length) return window.slice(); + return window.slice(window.length - newCount); +} + /** Matches the offset line in a stored summary, regardless of leading bullet. * Single source of truth for both detection (stampOffset) and extraction * (parseOffset) so the round-trip contract can't drift. */ diff --git a/src/hooks/wiki-worker.ts b/src/hooks/wiki-worker.ts index f40d955d..637b8552 100644 --- a/src/hooks/wiki-worker.ts +++ b/src/hooks/wiki-worker.ts @@ -19,7 +19,7 @@ const dlog = (msg: string) => _log("wiki-worker", msg); import { finalizeSummary, releaseLock, readState } from "./summary-state.js"; import { readSessionEventCache } from "./session-event-cache.js"; import { buildSessionPath } from "../utils/session-path.js"; -import { capLinesByBytes, stampOffset, WIKI_JSONL_MAX_BYTES } from "./wiki-offset.js"; +import { capLinesByBytes, newRowsFromWindow, stampOffset, WIKI_FALLBACK_MAX_ROWS, WIKI_JSONL_MAX_BYTES } from "./wiki-offset.js"; import { redactSecrets } from "./shared/redact.js"; import { uploadSummary } from "./upload-summary.js"; import { embedSummaryWithWarmup } from "../embeddings/embed-summary.js"; @@ -146,24 +146,38 @@ async function main(): Promise { // DB is still the source of truth: fall back to it whenever the cache is // absent (session resumed on another machine), empty, or — checked once // the offset is known — shorter than the offset already summarized. - const dbFetch = () => query( - `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + - `WHERE path LIKE '${esc(`/sessions/%${cfg.sessionId}%`)}' ORDER BY creation_date ASC` - ); + // Bounded DB fallback: the NEWEST WIKI_FALLBACK_MAX_ROWS rows (reversed to + // chronological) plus the true `total`. The old unbounded `ORDER BY ASC` + // materialized the whole fat `message` column (tens of MB, ~30s cold on a + // mega-session) even though only the newest un-summarized rows are consumed. + // `count(*)` reads no fat column, so `total` is cheap and lets the offset + // math (`newRowsFromWindow`) and the stamped offset stay correct. + const dbFetch = async (): Promise<{ rows: Record[]; total: number }> => { + const like = esc(`/sessions/%${cfg.sessionId}%`); + const cnt = await query(`SELECT count(*) AS n FROM "${cfg.sessionsTable}" WHERE path LIKE '${like}'`); + const total = Number(cnt[0]?.["n"] ?? 0); + if (total === 0) return { rows: [], total: 0 }; + const r = await query( + `SELECT message, creation_date FROM "${cfg.sessionsTable}" ` + + `WHERE path LIKE '${like}' ORDER BY creation_date DESC LIMIT ${WIKI_FALLBACK_MAX_ROWS}` + ); + return { rows: r.reverse(), total }; + }; // Retry on an empty result: the async capture writes (or Deeplake read // consistency) may simply be lagging behind SessionEnd under load. - const dbFetchWithRetry = async (): Promise[]> => { - let r = await dbFetch(); - for (let attempt = 1; r.length === 0 && attempt <= EVENT_FETCH_RETRIES; attempt++) { + const dbFetchWithRetry = async (): Promise<{ rows: Record[]; total: number }> => { + let f = await dbFetch(); + for (let attempt = 1; f.total === 0 && attempt <= EVENT_FETCH_RETRIES; attempt++) { const delay = EVENT_FETCH_BACKOFF_MS * attempt; wlog(`no events yet — retry ${attempt}/${EVENT_FETCH_RETRIES} in ${delay}ms`); await sleep(delay); - r = await dbFetch(); + f = await dbFetch(); } - return r; + return f; }; let usedLocalCache = false; + let dbTotal = 0; // true row count when the bounded DB path was used (else 0) let rows: Record[]; const cachedLines = readSessionEventCache(cfg.sessionId); if (cachedLines && cachedLines.length > 0) { @@ -172,7 +186,9 @@ async function main(): Promise { wlog(`loaded ${rows.length} events from local cache`); } else { wlog("fetching session events"); - rows = await dbFetchWithRetry(); + const f = await dbFetchWithRetry(); + rows = f.rows; + dbTotal = f.total; } if (rows.length === 0) { @@ -193,7 +209,9 @@ async function main(): Promise { return; } - let jsonlLines = rows.length; + // The offset high-water (stamped into the summary) must be the TRUE total, not the + // bounded window length — else the next run's offset regresses and re-summarizes. + let jsonlLines = usedLocalCache ? rows.length : dbTotal; // Derive the server path. When the events came from the local cache we've // already avoided the backend round-trip, so reproduce the canonical path @@ -263,15 +281,20 @@ async function main(): Promise { // to nothing, so re-load the full session from the DB instead. if (usedLocalCache && rows.length < prevOffset) { wlog(`local cache (${rows.length}) < summarized offset (${prevOffset}) — refetching from DB`); - rows = await dbFetchWithRetry(); - jsonlLines = rows.length; + const f = await dbFetchWithRetry(); + rows = f.rows; + dbTotal = f.total; + jsonlLines = dbTotal; + usedLocalCache = false; } // Feed claude only the rows added since the last summary. Reprocessing the // full session on every run is what drives ENOBUFS / 120s-timeout failures // on long (4000+ event) sessions — a stuck offset re-summarizes everything. // Reconstruct JSONL from individual rows (message is JSONB — may be object or string) - const newRows = prevOffset > 0 ? rows.slice(prevOffset) : rows; + const newRows = usedLocalCache + ? (prevOffset > 0 ? rows.slice(prevOffset) : rows) + : newRowsFromWindow(rows, dbTotal, prevOffset); if (prevOffset > 0 && newRows.length === 0) { wlog(`no new events since last summary (offset=${prevOffset}, total=${jsonlLines}) — skipping`); return; diff --git a/tests/claude-code/wiki-worker-local-cache.test.ts b/tests/claude-code/wiki-worker-local-cache.test.ts index fb0d102b..93be6fb1 100644 --- a/tests/claude-code/wiki-worker-local-cache.test.ts +++ b/tests/claude-code/wiki-worker-local-cache.test.ts @@ -99,11 +99,12 @@ function trackingFetch(sqls: string[], summaryRows: unknown[][] = []): void { const sql = JSON.parse(init.body).query as string; sqls.push(sql); if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: summaryRows }); + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[6]] }); if (sql.startsWith("SELECT message, creation_date")) { - // DB fallback path — a handful of generic rows. + // DB fallback path — a handful of generic rows (newest-first to match ORDER BY DESC). return jsonResp({ columns: ["message", "creation_date"], - rows: Array.from({ length: 6 }, (_, i) => [JSON.stringify({ type: "user_message", content: `db ${i}` }), "2026-01-01T00:00:00Z"]), + rows: Array.from({ length: 6 }, (_, i) => [JSON.stringify({ type: "user_message", content: `db ${5 - i}` }), "2026-01-01T00:00:00Z"]), }); } if (sql.startsWith("SELECT DISTINCT path")) { diff --git a/tests/claude-code/wiki-worker-plugin-version.test.ts b/tests/claude-code/wiki-worker-plugin-version.test.ts index 56b0f858..6dd4107a 100644 --- a/tests/claude-code/wiki-worker-plugin-version.test.ts +++ b/tests/claude-code/wiki-worker-plugin-version.test.ts @@ -127,6 +127,7 @@ async function runVariant(variant: AgentVariant, pluginVersion: string): Promise // Then SELECT summary FROM "memory" WHERE path = ... LIMIT 1 (to resume). fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], @@ -187,6 +188,7 @@ describe("wiki-worker pluginVersion threading — per agent", () => { process.argv[2] = configPath; fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], @@ -257,6 +259,7 @@ describe("wiki-worker API retry path — per agent", () => { let firstEventsCall = true; fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { if (firstEventsCall) { firstEventsCall = false; @@ -298,13 +301,15 @@ describe("wiki-worker resume + embeddings-disabled branches — per agent", () = process.argv[2] = configPath; fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[43]] }); if (sql.startsWith("SELECT message, creation_date")) { // 43 rows so the offset-42 resume baseline still leaves 1 new row to // summarize (otherwise the worker correctly skips a no-new-rows run). + // Newest-first (DESC) to match the real ORDER BY … DESC; worker reverses. return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 43 }, (_, i) => [ - JSON.stringify({ type: "user_message", content: `hi ${i}` }), + JSON.stringify({ type: "user_message", content: `hi ${42 - i}` }), "2026-04-20T00:00:00Z", ]), }); @@ -391,6 +396,7 @@ describe("wiki-worker error / edge-case branches — per agent", () => { // found" and exits early before invoking the LLM or upload path. fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[0]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [] }); } diff --git a/tests/claude-code/wiki-worker.test.ts b/tests/claude-code/wiki-worker.test.ts index 8139100c..14648adb 100644 --- a/tests/claude-code/wiki-worker.test.ts +++ b/tests/claude-code/wiki-worker.test.ts @@ -192,10 +192,13 @@ describe("wiki-worker — no events", () => { let eventSelects = 0; fetchMock.mockImplementation(async (_url: string, opts: any) => { const q = JSON.parse(opts.body).query as string; - if (/^\s*SELECT message, creation_date FROM "sessions"/.test(q)) { + // The bounded fallback retries on the cheap count probe now (not the fat + // message fetch): empty count for the first two attempts, then events appear. + if (/^\s*SELECT count\(\*\) AS n FROM "sessions"/.test(q)) { eventSelects++; - // Empty for the first two attempts, then the events appear. - if (eventSelects <= 2) return jsonResp({ columns: ["message", "creation_date"], rows: [] }); + return jsonResp({ columns: ["n"], rows: eventSelects <= 2 ? [[0]] : [[1]] }); + } + if (/^\s*SELECT message, creation_date FROM "sessions"/.test(q)) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi" }), "2026-01-01T00:00:00Z"]] }); } // path lookup + existing-summary lookup → empty is fine @@ -233,8 +236,14 @@ describe("wiki-worker — happy path", () => { let call = 0; return fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) { + return jsonResp({ columns: ["n"], rows: [[eventCount > 0 ? eventCount : eventRows.length]] }); + } if (sql.startsWith("SELECT message, creation_date")) { - return jsonResp({ columns: eventsCol, rows: eventCount > 0 ? manyRows : eventRows.map(r => [r.message, r.creation_date]) }); + // Worker fetches newest-first (ORDER BY creation_date DESC) then reverses back to + // chronological — the mock returns DESC to match the real DB. + const asc = eventCount > 0 ? manyRows : eventRows.map(r => [r.message, r.creation_date]); + return jsonResp({ columns: eventsCol, rows: asc.slice().reverse() }); } if (sql.startsWith("SELECT DISTINCT path")) { return jsonResp({ @@ -384,12 +393,14 @@ describe("wiki-worker — happy path", () => { it("serializes event rows that arrive as objects (JSONB) instead of strings", async () => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[2]] }); if (sql.startsWith("SELECT message, creation_date")) { + // DESC (newest-first) to match the real ORDER BY … DESC; worker reverses back. return jsonResp({ columns: ["message", "creation_date"], rows: [ - [{ type: "user_message", content: "hi" }, "2026-04-20T00:00:00Z"], [{ type: "tool_call", tool_name: "Bash" }, "2026-04-20T00:00:01Z"], + [{ type: "user_message", content: "hi" }, "2026-04-20T00:00:00Z"], ], }); } @@ -418,6 +429,7 @@ describe("wiki-worker — claude -p failure", () => { it("logs the claude exit code and skips the upload when no summary file lands", async () => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); @@ -438,6 +450,7 @@ describe("wiki-worker — claude -p failure", () => { it("falls back to err.message when err.status is absent", async () => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); @@ -506,6 +519,7 @@ describe("wiki-worker — finalize + release edge cases", () => { beforeEach(() => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); diff --git a/tests/codex/codex-wiki-worker.test.ts b/tests/codex/codex-wiki-worker.test.ts index 65d269d4..13ce5b4a 100644 --- a/tests/codex/codex-wiki-worker.test.ts +++ b/tests/codex/codex-wiki-worker.test.ts @@ -147,8 +147,12 @@ describe("codex wiki-worker — happy path", () => { ]); return fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[eventCount]] }); if (sql.startsWith("SELECT message, creation_date")) { - return jsonResp({ columns: ["message", "creation_date"], rows: eventCount === 1 ? eventRow.map(r => [r.message, r.creation_date]) : rows }); + // The worker now fetches newest-first (ORDER BY creation_date DESC) and reverses + // back to chronological, so the mock returns rows in DESC order to match the DB. + const asc = eventCount === 1 ? eventRow.map(r => [r.message, r.creation_date]) : rows; + return jsonResp({ columns: ["message", "creation_date"], rows: asc.slice().reverse() }); } if (sql.startsWith("SELECT DISTINCT path")) { return jsonResp({ @@ -291,6 +295,7 @@ describe("codex wiki-worker — happy path", () => { // — that would regenerate from scratch and overwrite the canonical summary. fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi" }), "t"]] }); } @@ -328,6 +333,7 @@ describe("codex wiki-worker — happy path", () => { rows: [[{ type: "user_message", content: "obj" }, "t"]], }); } + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); }); @@ -350,6 +356,7 @@ describe("codex wiki-worker — codex exec failure", () => { beforeEach(() => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); @@ -382,6 +389,7 @@ describe("codex wiki-worker — codex exec failure", () => { it("does not re-upload a stale existing summary after a failed regeneration", async () => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[8]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [["# Session X\n- **JSONL offset**: 7\n\n## What Happened\nprior"]] }); @@ -438,6 +446,7 @@ describe("codex wiki-worker — finalize + release edges", () => { beforeEach(() => { fetchMock.mockImplementation(async (_url: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); return jsonResp({ columns: ["summary"], rows: [] }); @@ -475,3 +484,154 @@ describe("codex wiki-worker — finalize + release edges", () => { expect(finalizeSummaryMock).not.toHaveBeenCalled(); }); }); + +const promptOf = (a: string[]) => a.find((x) => typeof x === "string" && x.includes("SUMMARY="))!; + +describe("codex wiki-worker — bounded-fetch edges + error paths (coverage)", () => { + function setupFetch(opts: { total: number; msgRows?: number; summaryOffset?: number }) { + const n = opts.msgRows ?? Math.min(opts.total, 2000); + const rows = Array.from({ length: n }, (_, i) => [JSON.stringify({ c: opts.total - 1 - i }), "t"]); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[opts.total]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: opts.summaryOffset === undefined ? [] : [[`# S\n- **JSONL offset**: ${opts.summaryOffset}\n\n## What Happened\nx`]] }); + return jsonResp({ columns: [], rows: [] }); + }); + } + const writesSummary = () => execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\ndone\n"); return Buffer.from(""); }); + + it("skips when the resume offset already covers every row", async () => { + setupFetch({ total: 5, summaryOffset: 5 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("no new events since last summary"); + }); + it("prefers the sidecar count over a smaller parsed offset", async () => { + setupFetch({ total: 10, summaryOffset: 3 }); readStateMock.mockReturnValue({ lastSummaryCount: 8 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + it("writes a NULL embedding when the embed daemon fails", async () => { + setupFetch({ total: 2 }); embedSummaryMock.mockRejectedValue(new Error("down")); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("summary embedding failed"); + }); + it("skips when the existing-summary lookup throws", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[2]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"], ["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) throw new Error("db down"); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs the sidecar update failure but still releases the lock", async () => { + setupFetch({ total: 2 }); writesSummary(); finalizeSummaryMock.mockImplementation(() => { throw new Error("boom"); }); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("sidecar update failed"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-codex"); + }); + it("swallows a releaseLock throw in the finally", async () => { + setupFetch({ total: 2 }); writesSummary(); releaseLockMock.mockImplementation(() => { throw new Error("boom"); }); + await expect(runWorker()).resolves.toBeUndefined(); + }); + it("truncates a single event exceeding the byte budget", async () => { + const huge = JSON.stringify({ c: "x".repeat(5 * 1024 * 1024) }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [[huge, "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("truncated it to stay within the buffer"); + }); + it("drops oldest rows when the batch exceeds the byte budget", async () => { + const row = JSON.stringify({ c: "y".repeat(600 * 1024) }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[10]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 10 }, () => [row, "t"]) }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("permanently skipping"); + }); + it("skips upload when exec throws AFTER a partial summary write", async () => { + setupFetch({ total: 2 }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "junk"); throw new Error("crash"); }); + await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs a fatal error and releases the lock when a query hard-fails", async () => { + fetchMock.mockResolvedValue(jsonResp("bad", false, 400)); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("fatal:"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-codex"); + }); + it("retries a retryable API error then succeeds", async () => { + vi.spyOn(global, "setTimeout").mockImplementation(((cb: any) => { cb(); return 0 as any; }) as any); + let first = true; + fetchMock.mockImplementation(async (_u: string, init: any) => { + if (first) { first = false; return jsonResp("busy", false, 503); } + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("retrying in"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); +}); + +describe("codex wiki-worker — unknown server path (coverage)", () => { + it("falls back to /sessions/unknown/ when the DISTINCT path lookup is empty", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\nok\n"); return Buffer.from(""); }); + await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + expect(releaseLockMock).toHaveBeenCalledWith("sid-codex"); + }); +}); + +describe("codex wiki-worker — formatExecFailure branches (coverage)", () => { + function setupOk() { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + } + it("formats an exec error carrying code + string stderr + Buffer stdout (with truncation)", async () => { + setupOk(); + const err: any = new Error("boom"); + err.code = "ETIMEDOUT"; + err.stderr = "e".repeat(400); // string branch + truncation + err.stdout = Buffer.from("o".repeat(400)); // Buffer branch + truncation + execFileSyncMock.mockImplementation(() => { throw err; }); + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("code=ETIMEDOUT"); + expect(log).toContain("stderr="); + expect(log).toContain("stdout="); + }); + it("formats an exec error with no recognizable fields as 'unknown failure'", async () => { + setupOk(); + execFileSyncMock.mockImplementation(() => { throw {}; }); + await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("unknown failure"); + }); +}); diff --git a/tests/cursor/cursor-wiki-worker.test.ts b/tests/cursor/cursor-wiki-worker.test.ts index 7d79ca09..11eea2ea 100644 --- a/tests/cursor/cursor-wiki-worker.test.ts +++ b/tests/cursor/cursor-wiki-worker.test.ts @@ -124,6 +124,7 @@ describe("cursor wiki-worker — behavior", () => { it("runs cursor-agent --print --force with the prompt as the trailing arg and uploads agent=cursor", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi cursor" }), "2026-04-20T00:00:00Z"]] }); } @@ -156,6 +157,7 @@ describe("cursor wiki-worker — behavior", () => { it("logs the failure and skips upload when the cursor-agent spawn throws", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi cursor" }), "2026-04-20T00:00:00Z"]] }); } @@ -207,6 +209,7 @@ describe("cursor wiki-worker — behavior", () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; sqls.push(sql); + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "db" }), "2026-04-20T00:00:00Z"]] }); } @@ -223,5 +226,131 @@ describe("cursor wiki-worker — behavior", () => { expect(sqls.some(s => s.startsWith("SELECT message, creation_date"))).toBe(true); expect(sqls.some(s => s.startsWith("SELECT DISTINCT path"))).toBe(true); + // Bounded fallback: cheap count probe + newest-N DESC LIMIT, never unbounded ASC. + expect(sqls.some(s => s.startsWith("SELECT count(*) AS n"))).toBe(true); + const fetchSql = sqls.find(s => s.startsWith("SELECT message, creation_date"))!; + expect(fetchSql).toContain("ORDER BY creation_date DESC"); + expect(fetchSql).toContain("LIMIT 2000"); + expect(sqls.some(s => s.includes("ORDER BY creation_date ASC"))).toBe(false); + }); +}); + +const promptOf = (a: string[]) => a.find((x) => typeof x === "string" && x.includes("SUMMARY="))!; + +describe("cursor wiki-worker — bounded-fetch edges + error paths (coverage)", () => { + function setupFetch(opts: { total: number; msgRows?: number; summaryOffset?: number }) { + const n = opts.msgRows ?? Math.min(opts.total, 2000); + const rows = Array.from({ length: n }, (_, i) => [JSON.stringify({ c: opts.total - 1 - i }), "t"]); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[opts.total]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: opts.summaryOffset === undefined ? [] : [[`# S\n- **JSONL offset**: ${opts.summaryOffset}\n\n## What Happened\nx`]] }); + return jsonResp({ columns: [], rows: [] }); + }); + } + const writesSummary = () => execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\ndone\n"); return Buffer.from(""); }); + + it("skips when the resume offset already covers every row", async () => { + setupFetch({ total: 5, summaryOffset: 5 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("no new events since last summary"); + }); + it("refetches from the bounded DB when the local cache is shorter than the offset", async () => { + readCacheMock.mockReturnValue(["a", "b"]); + const sqls: string[] = []; + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; sqls.push(sql); + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[43]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 43 }, (_, i) => [JSON.stringify({ t: 42 - i }), "t"]) }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: [["# S\n- **JSONL offset**: 40\n\n## What Happened\nx"]] }); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(sqls.find(s => s.startsWith("SELECT message, creation_date"))!).toContain("ORDER BY creation_date DESC"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + it("prefers the sidecar count over a smaller parsed offset", async () => { + setupFetch({ total: 10, summaryOffset: 3 }); readStateMock.mockReturnValue({ lastSummaryCount: 8 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + it("writes a NULL embedding when the embed daemon fails", async () => { + setupFetch({ total: 2 }); embedSummaryMock.mockRejectedValue(new Error("down")); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("summary embedding failed"); + }); + it("skips when the existing-summary lookup throws", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[2]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"], ["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) throw new Error("db down"); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs the sidecar update failure but still releases the lock", async () => { + setupFetch({ total: 2 }); writesSummary(); finalizeSummaryMock.mockImplementation(() => { throw new Error("boom"); }); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("sidecar update failed"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-cursor"); + }); + it("swallows a releaseLock throw in the finally", async () => { + setupFetch({ total: 2 }); writesSummary(); releaseLockMock.mockImplementation(() => { throw new Error("boom"); }); + await expect(runWorker()).resolves.toBeUndefined(); + }); + it("truncates a single event exceeding the byte budget", async () => { + const huge = JSON.stringify({ c: "x".repeat(5 * 1024 * 1024) }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [[huge, "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("truncated it to stay within the buffer"); + }); + it("drops oldest rows when the batch exceeds the byte budget", async () => { + const row = JSON.stringify({ c: "y".repeat(600 * 1024) }); + setupFetch({ total: 10, msgRows: 10 }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[10]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 10 }, () => [row, "t"]) }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("permanently skipping"); + }); + it("skips upload when exec throws AFTER a partial summary write", async () => { + setupFetch({ total: 2 }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "junk"); throw new Error("crash"); }); + await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs a fatal error and releases the lock when a query hard-fails", async () => { + fetchMock.mockResolvedValue(jsonResp("bad", false, 400)); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("fatal:"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-cursor"); + }); + it("retries a retryable API error then succeeds", async () => { + vi.spyOn(global, "setTimeout").mockImplementation(((cb: any) => { cb(); return 0 as any; }) as any); + let first = true; + fetchMock.mockImplementation(async (_u: string, init: any) => { + if (first) { first = false; return jsonResp("busy", false, 503); } + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("retrying in"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/hermes/hermes-wiki-worker.test.ts b/tests/hermes/hermes-wiki-worker.test.ts index 5dd5ffc5..c0bbf729 100644 --- a/tests/hermes/hermes-wiki-worker.test.ts +++ b/tests/hermes/hermes-wiki-worker.test.ts @@ -126,6 +126,9 @@ describe("hermes wiki-worker — behavior", () => { it("runs hermes -z --provider --yolo and uploads agent=hermes", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) { + return jsonResp({ columns: ["n"], rows: [[1]] }); + } if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi hermes" }), "2026-04-20T00:00:00Z"]] }); } @@ -158,6 +161,9 @@ describe("hermes wiki-worker — behavior", () => { it("logs the failure and skips upload when the hermes spawn throws", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) { + return jsonResp({ columns: ["n"], rows: [[1]] }); + } if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi hermes" }), "2026-04-20T00:00:00Z"]] }); } @@ -208,6 +214,7 @@ describe("hermes wiki-worker — behavior", () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; sqls.push(sql); + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "db" }), "2026-04-20T00:00:00Z"]] }); } @@ -224,5 +231,205 @@ describe("hermes wiki-worker — behavior", () => { expect(sqls.some(s => s.startsWith("SELECT message, creation_date"))).toBe(true); expect(sqls.some(s => s.startsWith("SELECT DISTINCT path"))).toBe(true); + // The fallback is BOUNDED now — a cheap count probe + newest-N DESC LIMIT, + // never the old unbounded `ORDER BY creation_date ASC` full fat-column scan. + expect(sqls.some(s => s.startsWith("SELECT count(*) AS n"))).toBe(true); + const fetchSql = sqls.find(s => s.startsWith("SELECT message, creation_date"))!; + expect(fetchSql).toContain("ORDER BY creation_date DESC"); + expect(fetchSql).toContain("LIMIT 2000"); + expect(sqls.some(s => s.includes("ORDER BY creation_date ASC"))).toBe(false); + }); +}); + +describe("hermes wiki-worker — bounded-fetch edges + error paths (coverage)", () => { + // Fetch mock: count(*) → total; message → newest-first rows; DISTINCT path; + // summary → optional existing (with offset). `summaryOffset` omitted → no summary. + function setupFetch(opts: { total: number; msgRows?: number; summaryOffset?: number }) { + const n = opts.msgRows ?? Math.min(opts.total, 2000); + const rows = Array.from({ length: n }, (_, i) => [ + JSON.stringify({ type: "user_message", content: `m${opts.total - 1 - i}` }), // DESC + "2026-04-20T00:00:00Z", + ]); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[opts.total]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) { + return jsonResp({ + columns: ["summary"], + rows: opts.summaryOffset === undefined ? [] : [[`# S\n- **JSONL offset**: ${opts.summaryOffset}\n\n## What Happened\nx`]], + }); + } + return jsonResp({ columns: [], rows: [] }); + }); + } + const writesSummary = () => + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { + writeFileSync(a[1].match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\ndone\n"); + return Buffer.from(""); + }); + + it("skips when the resume offset already covers every row (no new events)", async () => { + setupFetch({ total: 5, summaryOffset: 5 }); + writesSummary(); + await runWorker(); + expect(execFileSyncMock).not.toHaveBeenCalled(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("no new events since last summary"); + }); + + it("refetches from the bounded DB when the local cache is shorter than the offset", async () => { + readCacheMock.mockReturnValue(["line1", "line2"]); // 2 cached < offset 40 + const sqls: string[] = []; + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + sqls.push(sql); + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[43]] }); + if (sql.startsWith("SELECT message, creation_date")) + return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 43 }, (_, i) => [JSON.stringify({ t: 42 - i }), "t"]) }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: [["# S\n- **JSONL offset**: 40\n\n## What Happened\nx"]] }); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); + await runWorker(); + // The bounded DB fetch ran (count + newest-N DESC LIMIT), not the stale cache. + expect(sqls.some(s => s.startsWith("SELECT count(*) AS n"))).toBe(true); + const fetchSql = sqls.find(s => s.startsWith("SELECT message, creation_date"))!; + expect(fetchSql).toContain("ORDER BY creation_date DESC"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + + it("prefers the sidecar count over a smaller parsed offset", async () => { + setupFetch({ total: 10, summaryOffset: 3 }); + readStateMock.mockReturnValue({ lastSummaryCount: 8 }); // sidecar 8 > parsed 3 + writesSummary(); + await runWorker(); + // 10 total, offset 8 → 2 new rows summarized. + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + + it("writes a NULL embedding when the embed daemon fails, still uploads", async () => { + setupFetch({ total: 2 }); + embedSummaryMock.mockRejectedValue(new Error("embed daemon down")); + writesSummary(); + await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("summary embedding failed"); + }); + + it("skips the run when the existing-summary lookup throws (no overwrite)", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[2]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"], ["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) throw new Error("summary db down"); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); + await runWorker(); + expect(execFileSyncMock).not.toHaveBeenCalled(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + + it("logs the sidecar update failure but still releases the lock", async () => { + setupFetch({ total: 2 }); + writesSummary(); + finalizeSummaryMock.mockImplementation(() => { throw new Error("sidecar boom"); }); + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("sidecar update failed"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-hermes"); + }); + + it("swallows a releaseLock throw in the finally", async () => { + setupFetch({ total: 2 }); + writesSummary(); + releaseLockMock.mockImplementation(() => { throw new Error("release boom"); }); + await expect(runWorker()).resolves.toBeUndefined(); + }); +}); + +describe("hermes wiki-worker — more edges (coverage)", () => { + function setupFetch(total: number, msgRows: [unknown, string][]) { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[total]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: msgRows }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: [] }); + return jsonResp({ columns: [], rows: [] }); + }); + } + const writesSummary = () => + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { + writeFileSync(a[1].match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\ndone\n"); + return Buffer.from(""); + }); + + it("truncates a single event that exceeds the JSONL byte budget", async () => { + const huge = JSON.stringify({ type: "user_message", content: "x".repeat(5 * 1024 * 1024) }); + setupFetch(1, [[huge, "t"]]); + writesSummary(); + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("truncated it to stay within the buffer"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + + it("drops the oldest rows when the new batch exceeds the byte budget", async () => { + const row = JSON.stringify({ type: "user_message", content: "y".repeat(600 * 1024) }); + setupFetch(10, Array.from({ length: 10 }, () => [row, "t"]) as [unknown, string][]); + writesSummary(); + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("permanently skipping"); + }); + + it("skips the upload when the exec throws AFTER a partial summary write", async () => { + setupFetch(2, [["{}", "t"], ["{}", "t"]]); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { + writeFileSync(a[1].match(/SUMMARY=(\S+)/)![1], "partial junk"); + throw new Error("crashed mid-write"); + }); + await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("failed after a partial summary write"); + }); + + it("logs a fatal error and still releases the lock when a query hard-fails", async () => { + fetchMock.mockResolvedValue(jsonResp("bad request", false, 400)); // non-retryable → throws + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("fatal:"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-hermes"); + }); +}); + +describe("hermes wiki-worker — query retry (coverage)", () => { + it("retries a retryable API error (503) then succeeds", async () => { + vi.spyOn(global, "setTimeout").mockImplementation(((cb: any) => { cb(); return 0 as any; }) as any); + let first = true; + fetchMock.mockImplementation(async (_u: string, init: any) => { + if (first) { first = false; return jsonResp("busy", false, 503); } + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { + writeFileSync(a[1].match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\nok\n"); + return Buffer.from(""); + }); + await runWorker(); + const log = readFileSync(join(hooksDir, "wiki.log"), "utf-8"); + expect(log).toContain("retrying in"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/pi/pi-wiki-worker.test.ts b/tests/pi/pi-wiki-worker.test.ts index 709c9d13..8aaf85b7 100644 --- a/tests/pi/pi-wiki-worker.test.ts +++ b/tests/pi/pi-wiki-worker.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -119,6 +119,7 @@ describe("pi wiki-worker — behavior", () => { it("runs pi --print --provider --model with the prompt as the trailing arg and uploads agent=pi", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi pi" }), "2026-04-20T00:00:00Z"]] }); } @@ -152,6 +153,7 @@ describe("pi wiki-worker — behavior", () => { it("logs the failure and skips upload when the pi spawn throws", async () => { fetchMock.mockImplementation(async (_u: string, init: any) => { const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); if (sql.startsWith("SELECT message, creation_date")) { return jsonResp({ columns: ["message", "creation_date"], rows: [[JSON.stringify({ type: "user_message", content: "hi pi" }), "2026-04-20T00:00:00Z"]] }); } @@ -168,3 +170,123 @@ describe("pi wiki-worker — behavior", () => { expect(releaseLockMock).toHaveBeenCalledWith("sid-pi"); }); }); + +const promptOf = (a: string[]) => a.find((x) => typeof x === "string" && x.includes("SUMMARY="))!; + +describe("pi wiki-worker — bounded-fetch edges + error paths (coverage)", () => { + function setupFetch(opts: { total: number; msgRows?: number; summaryOffset?: number }) { + const n = opts.msgRows ?? Math.min(opts.total, 2000); + const rows = Array.from({ length: n }, (_, i) => [JSON.stringify({ c: opts.total - 1 - i }), "t"]); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[opts.total]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/sessions/alice/s.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) return jsonResp({ columns: ["summary"], rows: opts.summaryOffset === undefined ? [] : [[`# S\n- **JSONL offset**: ${opts.summaryOffset}\n\n## What Happened\nx`]] }); + return jsonResp({ columns: [], rows: [] }); + }); + } + const writesSummary = () => execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\ndone\n"); return Buffer.from(""); }); + + it("skips when the resume offset already covers every row", async () => { + setupFetch({ total: 5, summaryOffset: 5 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("no new events since last summary"); + }); + it("prefers the sidecar count over a smaller parsed offset", async () => { + setupFetch({ total: 10, summaryOffset: 3 }); readStateMock.mockReturnValue({ lastSummaryCount: 8 }); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); + it("writes a NULL embedding when the embed daemon fails", async () => { + setupFetch({ total: 2 }); embedSummaryMock.mockRejectedValue(new Error("down")); writesSummary(); await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("summary embedding failed"); + }); + it("skips when the existing-summary lookup throws", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[2]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"], ["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + if (sql.startsWith("SELECT summary FROM")) throw new Error("db down"); + return jsonResp({ columns: [], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs the sidecar update failure but still releases the lock", async () => { + setupFetch({ total: 2 }); writesSummary(); finalizeSummaryMock.mockImplementation(() => { throw new Error("boom"); }); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("sidecar update failed"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-pi"); + }); + it("swallows a releaseLock throw in the finally", async () => { + setupFetch({ total: 2 }); writesSummary(); releaseLockMock.mockImplementation(() => { throw new Error("boom"); }); + await expect(runWorker()).resolves.toBeUndefined(); + }); + it("truncates a single event exceeding the byte budget", async () => { + const huge = JSON.stringify({ c: "x".repeat(5 * 1024 * 1024) }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [[huge, "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("truncated it to stay within the buffer"); + }); + it("drops oldest rows when the batch exceeds the byte budget", async () => { + const row = JSON.stringify({ c: "y".repeat(600 * 1024) }); + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[10]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: Array.from({ length: 10 }, () => [row, "t"]) }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("permanently skipping"); + }); + it("skips upload when exec throws AFTER a partial summary write", async () => { + setupFetch({ total: 2 }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "junk"); throw new Error("crash"); }); + await runWorker(); + expect(uploadSummaryMock).not.toHaveBeenCalled(); + }); + it("logs a fatal error and releases the lock when a query hard-fails", async () => { + fetchMock.mockResolvedValue(jsonResp("bad", false, 400)); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("fatal:"); + expect(releaseLockMock).toHaveBeenCalledWith("sid-pi"); + }); + it("retries a retryable API error then succeeds", async () => { + vi.spyOn(global, "setTimeout").mockImplementation(((cb: any) => { cb(); return 0 as any; }) as any); + let first = true; + fetchMock.mockImplementation(async (_u: string, init: any) => { + if (first) { first = false; return jsonResp("busy", false, 503); } + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [["/x.jsonl"]] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + writesSummary(); await runWorker(); + expect(readFileSync(join(hooksDir, "wiki.log"), "utf-8")).toContain("retrying in"); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + }); +}); + +describe("pi wiki-worker — unknown server path (coverage)", () => { + it("falls back to /sessions/unknown/ when the DISTINCT path lookup is empty", async () => { + fetchMock.mockImplementation(async (_u: string, init: any) => { + const sql = JSON.parse(init.body).query as string; + if (sql.startsWith("SELECT count(*) AS n")) return jsonResp({ columns: ["n"], rows: [[1]] }); + if (sql.startsWith("SELECT message, creation_date")) return jsonResp({ columns: ["message", "creation_date"], rows: [["{}", "t"]] }); + if (sql.startsWith("SELECT DISTINCT path")) return jsonResp({ columns: ["path"], rows: [] }); + return jsonResp({ columns: ["summary"], rows: [] }); + }); + execFileSyncMock.mockImplementation((_b: string, a: string[]) => { writeFileSync(promptOf(a).match(/SUMMARY=(\S+)/)![1], "# S\n\n## What Happened\nok\n"); return Buffer.from(""); }); + await runWorker(); + expect(uploadSummaryMock).toHaveBeenCalledTimes(1); + expect(releaseLockMock).toHaveBeenCalledWith("sid-pi"); + }); +}); diff --git a/tests/shared/wiki-offset.test.ts b/tests/shared/wiki-offset.test.ts index 9fd45644..512f1d14 100644 --- a/tests/shared/wiki-offset.test.ts +++ b/tests/shared/wiki-offset.test.ts @@ -1,5 +1,11 @@ import { describe, it, expect } from "vitest"; -import { stampOffset, parseOffset, capLinesByBytes, WIKI_JSONL_MAX_BYTES } from "../../src/hooks/wiki-offset.js"; +import { + stampOffset, + parseOffset, + capLinesByBytes, + newRowsFromWindow, + WIKI_JSONL_MAX_BYTES, +} from "../../src/hooks/wiki-offset.js"; describe("stampOffset", () => { it("replaces an existing offset line, preserving the bullet, and round-trips via parseOffset", () => { @@ -68,3 +74,43 @@ describe("capLinesByBytes", () => { expect(truncated).toBe(false); }); }); + +describe("newRowsFromWindow — bounded-fetch offset math", () => { + const rows = (n: number, base = 0) => Array.from({ length: n }, (_, i) => base + i); + + it("full window (total == window length) equals a plain slice(prevOffset)", () => { + const w = rows(10); // whole session fetched (total 10) + expect(newRowsFromWindow(w, 10, 3)).toEqual(w.slice(3)); + expect(newRowsFromWindow(w, 10, 0)).toEqual(w); + expect(newRowsFromWindow(w, 10, 10)).toEqual([]); // nothing new + }); + + it("bounded window: returns the last (total - prevOffset) rows", () => { + // session has 5000 rows; we fetched the newest 2000; 4990 already summarized. + const w = rows(2000, 3000); // rows 3000..4999 (the newest 2000) + const out = newRowsFromWindow(w, 5000, 4990); // 10 new rows + expect(out).toHaveLength(10); + expect(out[0]).toBe(4990); + expect(out[9]).toBe(4999); + }); + + it("new rows exceed the window → returns the whole window (older-new fell outside the fetch)", () => { + // 5000-row session, newest 2000 fetched, but only 1000 summarized → 4000 'new', + // more than the 2000-row window. The 2000 older-new rows are outside the fetch — + // exactly the ones capLinesByBytes would drop; we summarize the newest 2000. + const w = rows(2000, 3000); + expect(newRowsFromWindow(w, 5000, 1000)).toEqual(w); + }); + + it("prevOffset 0 (regenerate) returns the whole window", () => { + const w = rows(2000, 3000); + expect(newRowsFromWindow(w, 5000, 0)).toEqual(w); + }); + + it("returns a copy, never the input array", () => { + const w = rows(3); + const out = newRowsFromWindow(w, 3, 0); + expect(out).toEqual(w); + expect(out).not.toBe(w); + }); +});