Skip to content

Commit 02b0834

Browse files
feat(mothership): expose table rows_version and schema hash to the copilot VFS snapshot
1 parent bfe8386 commit 02b0834

11 files changed

Lines changed: 292 additions & 54 deletions

apps/sim/lib/copilot/chat/workspace-context.test.ts

Lines changed: 95 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,22 @@ describe('buildWorkspaceMd - determinism (prompt-cache stability)', () => {
130130
{ id: 'wf-1', name: 'Alpha', isDeployed: true, folderPath: null },
131131
],
132132
tables: [
133-
{ id: 't-2', name: 'Orders', description: null, rowCount: 5 },
134-
{ id: 't-1', name: 'Customers', description: null, rowCount: 9 },
133+
{
134+
id: 't-2',
135+
name: 'Orders',
136+
description: null,
137+
rowCount: 5,
138+
rowsVersion: 2,
139+
schemaHash: 'bbb',
140+
},
141+
{
142+
id: 't-1',
143+
name: 'Customers',
144+
description: null,
145+
rowCount: 9,
146+
rowsVersion: 1,
147+
schemaHash: 'aaa',
148+
},
135149
],
136150
knowledgeBases: [
137151
{ id: 'kb-2', name: 'Docs', connectorTypes: ['notion', 'github'] },
@@ -187,8 +201,22 @@ describe('buildWorkspaceMd - determinism (prompt-cache stability)', () => {
187201
{ id: 'wf-2', name: 'Zeta', isDeployed: false, folderPath: null },
188202
],
189203
tables: [
190-
{ id: 't-1', name: 'Customers', description: null, rowCount: 9 },
191-
{ id: 't-2', name: 'Orders', description: null, rowCount: 5 },
204+
{
205+
id: 't-1',
206+
name: 'Customers',
207+
description: null,
208+
rowCount: 9,
209+
rowsVersion: 1,
210+
schemaHash: 'aaa',
211+
},
212+
{
213+
id: 't-2',
214+
name: 'Orders',
215+
description: null,
216+
rowCount: 5,
217+
rowsVersion: 2,
218+
schemaHash: 'bbb',
219+
},
192220
],
193221
knowledgeBases: [
194222
{ id: 'kb-1', name: 'Articles', connectorTypes: ['notion', 'github'] },
@@ -261,16 +289,77 @@ describe('buildWorkspaceMd - determinism (prompt-cache stability)', () => {
261289

262290
it('ignores volatile table row counts', () => {
263291
const a = buildWorkspaceMd(
264-
baseData({ tables: [{ id: 't-1', name: 'Customers', description: null, rowCount: 1 }] })
292+
baseData({
293+
tables: [
294+
{
295+
id: 't-1',
296+
name: 'Customers',
297+
description: null,
298+
rowCount: 1,
299+
rowsVersion: 1,
300+
schemaHash: 'aaa',
301+
},
302+
],
303+
})
265304
)
266305
const b = buildWorkspaceMd(
267-
baseData({ tables: [{ id: 't-1', name: 'Customers', description: null, rowCount: 9999 }] })
306+
baseData({
307+
tables: [
308+
{
309+
id: 't-1',
310+
name: 'Customers',
311+
description: null,
312+
rowCount: 9999,
313+
rowsVersion: 1,
314+
schemaHash: 'aaa',
315+
},
316+
],
317+
})
268318
)
269319
expect(a).toBe(b)
270320
expect(a).not.toContain('rows')
271321
})
272322
})
273323

324+
describe('buildVfsSnapshot - table versioning', () => {
325+
it('emits rowsVersion and schemaHash so row writes and schema edits change the snapshot', () => {
326+
const snap = buildVfsSnapshot(
327+
baseData({
328+
tables: [
329+
{
330+
id: 't-1',
331+
name: 'Customers',
332+
description: null,
333+
rowsVersion: 7,
334+
schemaHash: 'abc123def456',
335+
},
336+
],
337+
})
338+
)
339+
expect(snap.tables).toEqual([
340+
{ id: 't-1', name: 'Customers', rowsVersion: 7, schemaHash: 'abc123def456' },
341+
])
342+
})
343+
344+
it('omits a zero rowsVersion to mirror the Go contract omitempty semantics', () => {
345+
const snap = buildVfsSnapshot(
346+
baseData({
347+
tables: [
348+
{
349+
id: 't-1',
350+
name: 'Customers',
351+
description: null,
352+
rowsVersion: 0,
353+
schemaHash: 'abc123def456',
354+
},
355+
],
356+
})
357+
)
358+
expect(snap.tables?.[0]).not.toHaveProperty('rowsVersion')
359+
expect(snap.tables?.[0]).toHaveProperty('schemaHash', 'abc123def456')
360+
})
361+
})
362+
274363
describe('custom blocks', () => {
275364
const customBlocks = [
276365
{ type: 'custom_block_abc', name: 'Invoice Parser', description: 'Parses invoices' },

apps/sim/lib/copilot/chat/workspace-context.ts

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ import type {
2020
import { normalizeVfsSegment } from '@/lib/copilot/vfs/normalize-segment'
2121
import { canonicalWorkflowVfsDir, canonicalWorkspaceFilePath } from '@/lib/copilot/vfs/path-utils'
2222
import { getAccessibleOAuthCredentials } from '@/lib/credentials/environment'
23+
import { schemaFingerprint } from '@/lib/table/schema-fingerprint'
24+
import type { TableMetadata, TableSchema } from '@/lib/table/types'
2325
import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace'
2426
import { listCustomBlockSummariesForWorkspace } from '@/lib/workflows/custom-blocks/operations'
2527
import { listCustomTools } from '@/lib/workflows/custom-tools/operations'
@@ -67,7 +69,14 @@ export interface WorkspaceMdData {
6769
// prompt prefix); kept optional so callers that still have it cheaply (the VFS
6870
// materializer via listTables) need not change, while generateWorkspaceContext
6971
// skips the per-table COUNT query entirely.
70-
tables: Array<{ id: string; name: string; description?: string | null; rowCount?: number }>
72+
tables: Array<{
73+
id: string
74+
name: string
75+
description?: string | null
76+
rowCount?: number
77+
rowsVersion: number
78+
schemaHash: string
79+
}>
7180
files: Array<{ id: string; name: string; type: string; size: number; folderPath?: string | null }>
7281
oauthIntegrations: Array<{
7382
id: string
@@ -393,6 +402,9 @@ async function buildWorkspaceMdData(
393402
id: userTableDefinitions.id,
394403
name: userTableDefinitions.name,
395404
description: userTableDefinitions.description,
405+
schema: userTableDefinitions.schema,
406+
metadata: userTableDefinitions.metadata,
407+
rowsVersion: userTableDefinitions.rowsVersion,
396408
})
397409
.from(userTableDefinitions)
398410
.where(
@@ -497,7 +509,15 @@ async function buildWorkspaceMdData(
497509
// delta and needlessly bust the prompt cache.
498510
connectorTypes: connectorTypesByKb.get(kb.id)?.sort(stableCompare),
499511
})),
500-
tables: tables.map((t) => ({ id: t.id, name: t.name, description: t.description })),
512+
tables: tables.map((t) => ({
513+
id: t.id,
514+
name: t.name,
515+
description: t.description,
516+
rowsVersion: t.rowsVersion,
517+
// Raw stored schema + metadata: columnOrder lives in metadata, so a
518+
// pure reorder (metadata-only write) must still change the hash.
519+
schemaHash: schemaFingerprint(t.schema as TableSchema, t.metadata as TableMetadata | null),
520+
})),
501521
files: files.map((f) => ({
502522
id: f.id,
503523
name: f.name,
@@ -620,6 +640,10 @@ export function buildVfsSnapshot(data: WorkspaceMdData): VfsSnapshotV1 {
620640
id: t.id,
621641
name: t.name,
622642
...(t.description ? { description: t.description } : {}),
643+
// Omission mirrors the Go contract's omitempty so sim-emitted and
644+
// Go-remarshaled JSON stay byte-identical for the delta diff.
645+
...(t.rowsVersion ? { rowsVersion: t.rowsVersion } : {}),
646+
...(t.schemaHash ? { schemaHash: t.schemaHash } : {}),
623647
})),
624648
files: data.files.map((f) => ({
625649
id: f.id,

apps/sim/lib/copilot/generated/vfs-snapshot-v1.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ export interface VfsSnapshotV1Table {
107107
description?: string
108108
id: string
109109
name: string
110+
rowsVersion?: number
111+
schemaHash?: string
110112
}
111113
/**
112114
* This interface was referenced by `VfsSnapshotV1`'s JSON-Schema

apps/sim/lib/copilot/vfs/serializers.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ export function serializeTableMeta(table: {
319319
description?: string | null
320320
schema: unknown
321321
rowCount: number
322+
rowsVersion: number
322323
maxRows: number
323324
createdAt: Date | string
324325
updatedAt: Date | string
@@ -330,6 +331,7 @@ export function serializeTableMeta(table: {
330331
description: table.description || undefined,
331332
schema: table.schema,
332333
rowCount: table.rowCount,
334+
rowsVersion: table.rowsVersion,
333335
maxRows: table.maxRows,
334336
createdAt: table.createdAt instanceof Date ? table.createdAt.toISOString() : table.createdAt,
335337
updatedAt: table.updatedAt instanceof Date ? table.updatedAt.toISOString() : table.updatedAt,

apps/sim/lib/copilot/vfs/workspace-vfs.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ import { BINARY_DOC_TASKS, MAX_DOCUMENT_PREVIEW_CODE_BYTES } from '@/lib/executi
100100
import { runSandboxTask, SandboxUserCodeError } from '@/lib/execution/sandbox/run-task'
101101
import { getKnowledgeBases } from '@/lib/knowledge/service'
102102
import { validateMermaidSource } from '@/lib/mermaid/validate'
103+
import { schemaFingerprint } from '@/lib/table/schema-fingerprint'
103104
import { listTables } from '@/lib/table/service'
104105
import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager'
105106
import {
@@ -1647,6 +1648,7 @@ export class WorkspaceVFS {
16471648
description: table.description,
16481649
schema: table.schema,
16491650
rowCount: table.rowCount,
1651+
rowsVersion: table.rowsVersion,
16501652
maxRows: table.maxRows,
16511653
createdAt: table.createdAt,
16521654
updatedAt: table.updatedAt,
@@ -1659,6 +1661,8 @@ export class WorkspaceVFS {
16591661
name: t.name,
16601662
description: t.description,
16611663
rowCount: t.rowCount,
1664+
rowsVersion: t.rowsVersion,
1665+
schemaHash: schemaFingerprint(t.schema),
16621666
}))
16631667
} catch (err) {
16641668
logger.warn('Failed to materialize tables', {
@@ -2319,6 +2323,7 @@ export class WorkspaceVFS {
23192323
description: table.description,
23202324
schema: table.schema,
23212325
rowCount: table.rowCount,
2326+
rowsVersion: table.rowsVersion,
23222327
maxRows: table.maxRows,
23232328
createdAt: table.createdAt,
23242329
updatedAt: table.updatedAt,

apps/sim/lib/table/column-keys.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import type {
1414
Filter,
1515
RowData,
1616
Sort,
17+
TableMetadata,
1718
TableSchema,
1819
WorkflowGroup,
1920
} from '@/lib/table/types'
@@ -40,6 +41,37 @@ export function generateColumnId(): string {
4041
return `col_${generateId().replace(/-/g, '')}`
4142
}
4243

44+
/**
45+
* Returns `schema` with `columns` sorted by `metadata.columnOrder` (the user-
46+
* editable visible order). Columns missing from `columnOrder` are appended at
47+
* the end in their original (schema-creation) order — covers tables created
48+
* before `columnOrder` existed and any drift from out-of-band column adds.
49+
*
50+
* This makes `schema.columns` the single source of truth for column order on
51+
* the wire. The client doesn't have to join the two arrays itself — every
52+
* consumer (grid, sidebar, copilot, mothership) gets the same ordered list.
53+
*/
54+
export function applyColumnOrderToSchema(
55+
schema: TableSchema,
56+
metadata: TableMetadata | null
57+
): TableSchema {
58+
const order = metadata?.columnOrder
59+
if (!order || order.length === 0) return schema
60+
// `columnOrder` holds stable column ids (legacy entries equal the name == id).
61+
const byId = new Map<string, TableSchema['columns'][number]>()
62+
for (const c of schema.columns) byId.set(getColumnId(c), c)
63+
const ordered: TableSchema['columns'] = []
64+
for (const id of order) {
65+
const c = byId.get(id)
66+
if (c) {
67+
ordered.push(c)
68+
byId.delete(id)
69+
}
70+
}
71+
for (const c of byId.values()) ordered.push(c)
72+
return { ...schema, columns: ordered }
73+
}
74+
4375
/**
4476
* Matches a column against a reference that may be a stable id (first-party
4577
* callers) or a display name (legacy / mothership / public API). Id match is
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { describe, expect, it } from 'vitest'
2+
import { schemaFingerprint } from '@/lib/table/schema-fingerprint'
3+
import type { TableSchema } from '@/lib/table/types'
4+
5+
function schema(columns: Array<{ id?: string; name: string }>): TableSchema {
6+
return { columns: columns.map((c) => ({ ...c, type: 'string' })) } as TableSchema
7+
}
8+
9+
describe('schemaFingerprint', () => {
10+
it('is stable for the same column shape', () => {
11+
const a = schemaFingerprint(schema([{ id: 'col_1', name: 'email' }]))
12+
const b = schemaFingerprint(schema([{ id: 'col_1', name: 'email' }]))
13+
expect(a).toBe(b)
14+
expect(a).toMatch(/^[0-9a-f]{12}$/)
15+
})
16+
17+
it('changes on rename, add, and reorder', () => {
18+
const base = schemaFingerprint(
19+
schema([
20+
{ id: 'col_1', name: 'email' },
21+
{ id: 'col_2', name: 'age' },
22+
])
23+
)
24+
const renamed = schemaFingerprint(
25+
schema([
26+
{ id: 'col_1', name: 'contact' },
27+
{ id: 'col_2', name: 'age' },
28+
])
29+
)
30+
const added = schemaFingerprint(
31+
schema([
32+
{ id: 'col_1', name: 'email' },
33+
{ id: 'col_2', name: 'age' },
34+
{ id: 'col_3', name: 'city' },
35+
])
36+
)
37+
const reordered = schemaFingerprint(
38+
schema([
39+
{ id: 'col_2', name: 'age' },
40+
{ id: 'col_1', name: 'email' },
41+
])
42+
)
43+
expect(new Set([base, renamed, added, reordered]).size).toBe(4)
44+
})
45+
46+
it('keys legacy columns without ids by name (getColumnId fallback)', () => {
47+
const legacy = schemaFingerprint(schema([{ name: 'email' }]))
48+
const withId = schemaFingerprint(schema([{ id: 'col_1', name: 'email' }]))
49+
expect(legacy).not.toBe(withId)
50+
})
51+
52+
it('changes on a pure metadata.columnOrder reorder of the RAW schema', () => {
53+
// The user-visible order lives in metadata.columnOrder and is written by a
54+
// metadata-only update (no schema write, no rows_version bump) — the hash
55+
// is the ONLY signal such a reorder can move, so it must move.
56+
const raw = schema([
57+
{ id: 'col_1', name: 'email' },
58+
{ id: 'col_2', name: 'age' },
59+
])
60+
const unordered = schemaFingerprint(raw, null)
61+
const reordered = schemaFingerprint(raw, { columnOrder: ['col_2', 'col_1'] })
62+
expect(reordered).not.toBe(unordered)
63+
64+
// Raw schema + metadata order must hash identically to an already
65+
// order-applied schema (getTableById/listTables output) without metadata —
66+
// otherwise the same table carries two hashes across call sites.
67+
const applied = schemaFingerprint(
68+
schema([
69+
{ id: 'col_2', name: 'age' },
70+
{ id: 'col_1', name: 'email' },
71+
])
72+
)
73+
expect(reordered).toBe(applied)
74+
})
75+
76+
it('matches the golden hash (pins the storage-key format across refactors)', () => {
77+
// Snapshot-cache storage keys embed this hash
78+
// (table-snapshots/{ws}/{tableId}/v{version}-{hash}.csv). Changing the
79+
// hashed shape orphans every cached CSV and emits a one-time '~table'
80+
// delta for every table in every live chat — if this assertion fails,
81+
// that blast radius is intentional and reviewed, or the change is wrong.
82+
const golden = schemaFingerprint(
83+
schema([
84+
{ id: 'col_1', name: 'email' },
85+
{ id: 'col_2', name: 'age' },
86+
])
87+
)
88+
expect(golden).toBe('f49aa06b1b7c')
89+
})
90+
})

0 commit comments

Comments
 (0)