Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions packages/app/cypress/e2e/evaluation-chart.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,103 @@ describe('Evaluation Chart — Content & Interactions', () => {
.should('be.checked');
});
});

describe('Evaluation sample sharing', () => {
beforeEach(() => {
cy.intercept('GET', '/api/v1/eval-samples*', {
statusCode: 200,
body: {
samples: [
{
docId: 0,
prompt: 'What is 1 + 1?',
target: '2',
response: '2',
rawResponse: null,
demonstrations: null,
passed: true,
score: 1,
metrics: {},
},
],
total: 1,
passedTotal: 1,
failedTotal: 0,
source: 'db',
offset: 0,
},
});
cy.visit('/evaluation');
cy.get('[data-testid="evaluation-chart-display"]').should('be.visible');
cy.get('[data-testid="evaluation-view-toggle"]').contains('Table').click();
});

it('copies and restores a link to the prompt drawer', () => {
cy.on(
'uncaught:exception',
(error) =>
!error.message.includes('Hydration failed') &&
!error.message.includes('Minified React error #418'),
);
cy.get('[title="View per-sample prompts and responses"]').first().click();

cy.window().then((win) => {
cy.stub(win.navigator.clipboard, 'writeText').as('writeDrawerLink').resolves();
});
cy.get('[data-testid="eval-drawer-share-button"]').click();
cy.contains('[data-testid="eval-drawer-share-button"]', 'Copied').should('be.visible');

cy.get('@writeDrawerLink')
.should('have.been.calledOnce')
.then((stub) => {
const sharedUrl = String((stub as sinon.SinonStub).firstCall.args[0]);
const params = new URL(sharedUrl).searchParams;
expect(params.get('eval')).to.match(/^\d+$/);
expect(params.has('sample')).to.equal(false);
cy.visit(sharedUrl);
});

cy.get('[role="dialog"]').should('be.visible');
cy.get('[data-testid="eval-drawer-share-button"]').should('be.visible');
cy.get('[role="dialog"] li > button[aria-expanded="true"]').should('not.exist');
});

it('does not apply a stale sample id to a manually opened drawer', () => {
cy.visit('/evaluation?sample=0');
cy.get('[data-testid="evaluation-chart-display"]').should('be.visible');
cy.get('[title="View per-sample prompts and responses"]').first().click();

cy.get('[role="dialog"]').should('be.visible');
cy.get('[role="dialog"] li > button[aria-expanded="true"]').should('not.exist');
});

it('copies and restores a link to one expanded sample', () => {
cy.on(
'uncaught:exception',
(error) =>
!error.message.includes('Hydration failed') &&
!error.message.includes('Minified React error #418'),
);
cy.get('[title="View per-sample prompts and responses"]').first().click();
cy.get('[role="dialog"] li > button').first().click();

cy.window().then((win) => {
cy.stub(win.navigator.clipboard, 'writeText').as('writeShareLink').resolves();
});
cy.get('[data-testid^="eval-sample-share-"]').click();
cy.contains('[data-testid^="eval-sample-share-"]', 'Copied').should('be.visible');

cy.get('@writeShareLink')
.should('have.been.calledOnce')
.then((stub) => {
const sharedUrl = String((stub as sinon.SinonStub).firstCall.args[0]);
expect(new URL(sharedUrl).searchParams.get('eval')).to.match(/^\d+$/);
expect(new URL(sharedUrl).searchParams.get('sample')).to.match(/^\d+$/);
cy.visit(sharedUrl);
});

cy.get('[role="dialog"]').should('be.visible');
cy.get('[aria-expanded="true"]').should('exist');
cy.get('[data-testid^="eval-sample-share-"]').scrollIntoView().should('be.visible');
});
});
28 changes: 24 additions & 4 deletions packages/app/src/app/api/v1/eval-samples/route.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import { type NextRequest, NextResponse } from 'next/server';

import { getDb } from '@semianalysisai/inferencex-db/connection';
import { getEvalSamples } from '@semianalysisai/inferencex-db/queries/eval-samples';
import {
getEvalSampleOffset,
getEvalSamples,
} from '@semianalysisai/inferencex-db/queries/eval-samples';

import { cachedJson, cachedQuery } from '@/lib/api-cache';
import { extractDemonstrations } from '@/lib/eval-sample-utils';
Expand All @@ -17,9 +20,13 @@ const getCachedEvalSamples = cachedQuery(
getEvalSamples(getDb(), evalResultId, filter, offset, limit),
'eval-samples',
);
const getCachedEvalSampleOffset = cachedQuery(
(evalResultId: number, docId: number) => getEvalSampleOffset(getDb(), evalResultId, docId),
'eval-sample-offset',
);

/**
* GET /api/v1/eval-samples?eval_result_id=N&filter=all|passed|failed&offset=0&limit=200
* GET /api/v1/eval-samples?eval_result_id=N&filter=all|passed|failed&offset=0&limit=200&doc_id=N
*
* Returns a paginated slice of per-prompt samples for one `eval_results` row,
* plus passed/failed totals for the filter-chip badges. Drawer use only —
Expand All @@ -33,6 +40,8 @@ export async function GET(request: NextRequest) {
const evalResultId = Number(params.get('eval_result_id'));
const filterParam = params.get('filter') ?? 'all';
const offset = Math.max(0, Math.trunc(Number(params.get('offset') ?? '0')));
const docIdParam = params.get('doc_id');
const docId = docIdParam === null ? null : Math.trunc(Number(docIdParam));
const requestedLimit = Math.trunc(Number(params.get('limit') ?? String(DEFAULT_LIMIT)));
const limit = Math.min(MAX_LIMIT, Math.max(1, requestedLimit || DEFAULT_LIMIT));

Expand All @@ -42,6 +51,9 @@ export async function GET(request: NextRequest) {
{ status: 400 },
);
}
if (docId !== null && (!Number.isFinite(docId) || docId < 0)) {
return NextResponse.json({ error: 'doc_id must be a non-negative integer' }, { status: 400 });
}
if (!ALLOWED_FILTERS.has(filterParam)) {
return NextResponse.json(
{ error: `filter must be one of: ${[...ALLOWED_FILTERS].join(', ')}` },
Expand All @@ -51,8 +63,15 @@ export async function GET(request: NextRequest) {
const filter = filterParam as 'all' | 'passed' | 'failed';

try {
const result = await getCachedEvalSamples(evalResultId, filter, offset, limit);

const sampleOffset =
docId === null ? null : await getCachedEvalSampleOffset(evalResultId, docId);
if (docId !== null && sampleOffset === null) {
return NextResponse.json({ error: 'Sample not found' }, { status: 404 });
}
const resolvedOffset =
sampleOffset === null ? offset : Math.floor(sampleOffset / limit) * limit;
const resolvedFilter = docId === null ? filter : 'all';
const result = await getCachedEvalSamples(evalResultId, resolvedFilter, resolvedOffset, limit);
return cachedJson({
samples: result.samples.map((s) => ({
docId: s.doc_id,
Expand All @@ -69,6 +88,7 @@ export async function GET(request: NextRequest) {
passedTotal: result.passedTotal,
failedTotal: result.failedTotal,
source: 'db' as const,
offset: resolvedOffset,
});
} catch (error) {
console.error('Error fetching eval samples:', error);
Expand Down
102 changes: 97 additions & 5 deletions packages/app/src/components/evaluation/ui/EvalSamplesDrawer.tsx
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
'use client';

import { ChevronLeft, ChevronRight, Search } from 'lucide-react';
import { useEffect, useMemo, useState } from 'react';
import { Check, ChevronLeft, ChevronRight, Search, Share2 } from 'lucide-react';
import { useEffect, useMemo, useRef, useState } from 'react';

import type { EvaluationChartData } from '@/components/evaluation/types';
import { Dialog, DialogContent, DialogTitle } from '@/components/ui/dialog';
import { useEvalSamples } from '@/hooks/api/use-eval-samples';
import { track } from '@/lib/analytics';
import type { EvalSamplesFilter, EvalSamplesLiveContext } from '@/lib/api';
import { buildShareUrl } from '@/lib/url-state';
import { useLocale } from '@/lib/use-locale';

const PAGE_SIZE = 50;
Expand Down Expand Up @@ -44,6 +45,8 @@ const STRINGS = {
total === 0 ? '0 of 0' : `${start}–${end} of ${total}`,
prevPage: 'Previous page',
nextPage: 'Next page',
shareSample: 'Share',
copied: 'Copied',
},
zh: {
score: '得分',
Expand Down Expand Up @@ -73,13 +76,17 @@ const STRINGS = {
total === 0 ? '共 0 条' : `第 ${start}–${end} 条,共 ${total} 条`,
prevPage: '上一页',
nextPage: '下一页',
shareSample: '分享',
copied: '已复制',
},
} as const;

interface EvalSamplesDrawerProps {
/** The selected row from the EvaluationTable, or null when closed. */
row: EvaluationChartData | null;
onClose: () => void;
/** Sample selected by a shared URL. */
initialDocId?: number | null;
}

/**
Expand All @@ -91,14 +98,21 @@ interface EvalSamplesDrawerProps {
* Inspired by the vLLM eval dashboard PoC
* (credit: @khluu, @simon-mo, @robertgshaw2-redhat).
*/
export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerProps) {
export default function EvalSamplesDrawer({
row,
onClose,
initialDocId = null,
}: EvalSamplesDrawerProps) {
const open = row !== null;
const locale = useLocale();
const t = STRINGS[locale];
const [filter, setFilter] = useState<EvalSamplesFilter>('all');
const [page, setPage] = useState(0);
const [search, setSearch] = useState('');
const [expanded, setExpanded] = useState<Set<number>>(new Set());
const [requestedDocId, setRequestedDocId] = useState<number | null>(initialDocId);
const [copiedTarget, setCopiedTarget] = useState<number | 'drawer' | null>(null);
const bodyRef = useRef<HTMLDivElement>(null);

// Reset transient state whenever a new row is opened.
useEffect(() => {
Expand All @@ -107,7 +121,9 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
setPage(0);
setSearch('');
setExpanded(new Set());
}, [row?.evalResultId, open]);
setRequestedDocId(initialDocId);
setCopiedTarget(null);
}, [row?.evalResultId, open, initialDocId]);

// Build a live-fetch context for unofficial runs from the row's identifying
// fields. The hook ignores this when `evalResultId > 0` (DB-backed path).
Expand All @@ -134,6 +150,7 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
filter,
offset: page * PAGE_SIZE,
limit: PAGE_SIZE,
docId: requestedDocId,
});

// Client-side substring filter on the page slice — server-side full-text
Expand All @@ -156,6 +173,19 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE));
const safePage = Math.min(page, totalPages - 1);

useEffect(() => {
if (requestedDocId === null || !data || data.offset === undefined) return;
if (!data.samples.some((sample) => sample.docId === requestedDocId)) return;
setPage(Math.floor(data.offset / PAGE_SIZE));
setExpanded(new Set([requestedDocId]));
requestAnimationFrame(() => {
bodyRef.current
?.querySelector<HTMLElement>(`[data-eval-sample-id="${requestedDocId}"]`)
?.scrollIntoView({ block: 'center' });
});
setRequestedDocId(null);
}, [data, requestedDocId]);
Comment thread
adibarra marked this conversation as resolved.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shared sample errors stick drawer

Medium Severity

requestedDocId is cleared only after a successful response that contains the sample. When the API returns 404 for a missing doc_id, or the sample never appears in data, that id stays set, so later page and filter requests keep sending it and the drawer remains stuck on the load error until it is closed and reopened.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 555aae7. Configure here.


const handleFilterChange = (next: EvalSamplesFilter) => {
setFilter(next);
setPage(0);
Expand All @@ -177,6 +207,33 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
track('evaluation_samples_paged', { direction: delta > 0 ? 'next' : 'prev' });
};

const handleShare = async (docId?: number) => {
if (!row || row.evalResultId <= 0) return;
const url = new URL(buildShareUrl());
url.searchParams.set('eval', String(row.evalResultId));
if (docId === undefined) url.searchParams.delete('sample');
else url.searchParams.set('sample', String(docId));

try {
await navigator.clipboard.writeText(url.toString());
} catch {
const textArea = document.createElement('textarea');
textArea.value = url.toString();
document.body.append(textArea);
textArea.select();
document.execCommand('copy');
textArea.remove();
}

const target = docId ?? 'drawer';
setCopiedTarget(target);
setTimeout(() => setCopiedTarget((current) => (current === target ? null : current)), 2000);
track(docId === undefined ? 'evaluation_drawer_link_copied' : 'evaluation_sample_link_copied', {
eval_result_id: row.evalResultId,
...(docId === undefined ? {} : { doc_id: docId }),
});
};

const isUnofficial = row !== null && row.evalResultId <= 0;
// Unofficial runs are renderable as long as we resolved a runId from the row's url.
// If we couldn't (no run_url, malformed url, etc.), fall back to the empty-state copy below.
Expand Down Expand Up @@ -229,6 +286,23 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
</div>
)}
</div>
{row && row.evalResultId > 0 && (
<button
type="button"
onClick={() => handleShare()}
className="inline-flex shrink-0 items-center gap-1 rounded-md border border-border px-2 py-1 text-xs font-medium text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
aria-label={t.shareSample}
title={t.shareSample}
data-testid="eval-drawer-share-button"
>
{copiedTarget === 'drawer' ? (
<Check className="size-3.5" />
) : (
<Share2 className="size-3.5" />
)}
{copiedTarget === 'drawer' ? t.copied : t.shareSample}
</button>
)}
</div>

{/* Filter chips + search */}
Expand Down Expand Up @@ -267,7 +341,7 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
</div>

{/* Body */}
<div className="overflow-auto px-4 py-3">
<div ref={bodyRef} className="overflow-auto px-4 py-3">
{liveUnavailable && (
<p className="rounded-md border border-amber-500/40 bg-amber-500/5 px-3 py-2 text-xs text-amber-700 dark:text-amber-400">
{t.liveUnavailable}
Expand Down Expand Up @@ -300,6 +374,7 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
{filteredSamples.map((s) => (
<li
key={s.docId}
data-eval-sample-id={s.docId}
className="rounded-md border border-border/70 bg-card/30 transition-colors hover:bg-card/50"
>
<button
Expand Down Expand Up @@ -343,6 +418,23 @@ export default function EvalSamplesDrawer({ row, onClose }: EvalSamplesDrawerPro
emptyText={t.empty}
/>
)}
{row && row.evalResultId > 0 && (
<button
type="button"
onClick={() => handleShare(s.docId)}
className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[11px] font-medium text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
aria-label={t.shareSample}
title={t.shareSample}
data-testid={`eval-sample-share-${s.docId}`}
>
{copiedTarget === s.docId ? (
<Check className="size-3.5" />
) : (
<Share2 className="size-3.5" />
)}
{copiedTarget === s.docId ? t.copied : t.shareSample}
</button>
)}
</div>
)}
</li>
Expand Down
Loading
Loading