diff --git a/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts b/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts new file mode 100644 index 00000000000..0afa418d05b --- /dev/null +++ b/apps/sim/lib/execution/payloads/file-secret-provenance.test.ts @@ -0,0 +1,124 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { metadata, readWorkspaceFile } = vi.hoisted(() => ({ + metadata: vi.fn(), + readWorkspaceFile: vi.fn(), +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataByKey: metadata })) +vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ + readWorkspaceFileRecordByKey: { execute: readWorkspaceFile }, +})) + +import { resolveStoredFileProvenanceSource } from '@/lib/execution/payloads/file-secret-provenance' + +const context = { + principal: { kind: 'session', userId: 'reader', sessionId: 'session' } as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} +const file = { + key: 'execution/workspace-1/workflow-1/execution-1/unique/archive.zip', + context: 'execution' as const, +} +const revision = new Date('2026-09-11T00:00:00.000Z') +const record = { + id: 'canonical-file', + key: file.key, + context: 'execution', + workspaceId: context.workspaceId, + userId: 'writer', + contentUpdatedAt: revision, +} + +describe('stored file provenance source', () => { + beforeEach(() => { + vi.clearAllMocks() + metadata.mockResolvedValue(record) + readWorkspaceFile.mockResolvedValue({ file: {} }) + }) + + it('uses the canonical execution file identity and revision', async () => { + expect(await resolveStoredFileProvenanceSource(file, context)).toEqual({ + identity: { + fileId: record.id, + key: record.key, + context: 'execution', + contentUpdatedAt: revision, + }, + ownerUserId: 'writer', + }) + expect(metadata).toHaveBeenCalledWith(file.key, undefined, { includeDeleted: true }) + }) + + it.each([ + { workspaceId: 'foreign-workspace' }, + { workflowId: 'foreign-workflow' }, + { executionId: 'foreign-execution' }, + ])('refuses an out-of-scope file before metadata lookup: %j', async (scope) => { + await expect(resolveStoredFileProvenanceSource(file, { ...context, ...scope })).rejects.toThrow( + 'File not found' + ) + expect(metadata).not.toHaveBeenCalled() + }) + + it('accepts a causally inherited file key in the same workflow', async () => { + await expect( + resolveStoredFileProvenanceSource(file, { + ...context, + executionId: 'resumed-execution', + fileKeys: [file.key], + }) + ).resolves.toMatchObject({ identity: { fileId: 'canonical-file' } }) + }) + + it('does not let a file key allowlist cross workspaces', async () => { + await expect( + resolveStoredFileProvenanceSource(file, { + ...context, + workspaceId: 'foreign-workspace', + fileKeys: [file.key], + }) + ).rejects.toThrow('File not found') + expect(metadata).not.toHaveBeenCalled() + }) + + it('rejects a forged context before metadata lookup', async () => { + await expect( + resolveStoredFileProvenanceSource({ ...file, context: 'workspace' }, context) + ).rejects.toThrow('File context does not match its storage key') + expect(metadata).not.toHaveBeenCalled() + }) + + it.each([ + { workspaceId: 'foreign-workspace' }, + { context: 'workspace' }, + { context: 'knowledge-base' }, + ])('rejects mismatched canonical metadata: %j', async (changes) => { + metadata.mockResolvedValue({ ...record, ...changes }) + await expect(resolveStoredFileProvenanceSource(file, context)).rejects.toThrow('File not found') + }) + + it('preserves a missing legacy record as absence', async () => { + metadata.mockResolvedValue(null) + await expect(resolveStoredFileProvenanceSource(file, context)).resolves.toBeUndefined() + }) + + it('does not turn metadata lookup failures into legacy absence', async () => { + metadata.mockRejectedValue(new Error('database unavailable')) + await expect(resolveStoredFileProvenanceSource(file, context)).rejects.toThrow( + 'database unavailable' + ) + }) + + it('requires the workspace file use case before resolving a workspace source', async () => { + const workspaceFile = { key: 'workspace/workspace-1/file.txt', context: 'workspace' as const } + readWorkspaceFile.mockRejectedValue(new Error('Workspace access denied')) + await expect(resolveStoredFileProvenanceSource(workspaceFile, context)).rejects.toThrow( + 'Workspace access denied' + ) + expect(metadata).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/execution/payloads/file-secret-provenance.ts b/apps/sim/lib/execution/payloads/file-secret-provenance.ts new file mode 100644 index 00000000000..1af18b999b3 --- /dev/null +++ b/apps/sim/lib/execution/payloads/file-secret-provenance.ts @@ -0,0 +1,61 @@ +import type { Principal } from '@sim/auth/principal' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + assertUserFileContentAccess, + ExecutionFileAccessError, + type ExecutionMaterializationContext, +} from '@/lib/execution/payloads/materialization.server' +import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' +import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' +import type { UserFile } from '@/executor/types' + +export interface StoredFileProvenanceSource { + identity: WorkspaceFileSecretProvenanceIdentity + ownerUserId: string +} + +/** + * Binds an authorized stored-file read to its canonical content revision. Execution callers pass + * the same trusted run capability used to read the bytes; a file object's id is never an identity. + * Files predating metadata registration retain the caller's existing absence policy. + */ +export async function resolveStoredFileProvenanceSource( + file: Pick, + context: ExecutionMaterializationContext & { principal: Principal; workspaceId: string } +): Promise { + if (!file.key) return undefined + try { + await assertUserFileContentAccess(file, context) + } catch (error) { + if (error instanceof ExecutionFileAccessError) { + throw new OrchestrationError('not_found', 'File not found') + } + throw error + } + const storageContext = inferContextFromKey(file.key) + if (storageContext !== 'workspace' && storageContext !== 'execution') return undefined + + const metadata = await getFileMetadataByKey(file.key, undefined, { includeDeleted: true }) + if (!metadata) return undefined + if ( + (metadata.context !== 'workspace' && + metadata.context !== 'mothership' && + metadata.context !== 'execution') || + metadata.workspaceId !== context.workspaceId || + (storageContext === 'execution' + ? metadata.context !== 'execution' + : metadata.context !== 'workspace' && metadata.context !== 'mothership') + ) { + throw new OrchestrationError('not_found', 'File not found') + } + return { + identity: { + fileId: metadata.id, + key: metadata.key, + context: metadata.context, + contentUpdatedAt: metadata.contentUpdatedAt, + }, + ownerUserId: metadata.userId, + } +} diff --git a/apps/sim/lib/execution/payloads/materialization.server.test.ts b/apps/sim/lib/execution/payloads/materialization.server.test.ts index cdcdcd225d3..c7208738e04 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.test.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockDownloadServableFileFromStorage, mockReadWorkspaceFileByKey, mockVerifyFileAccess } = @@ -22,7 +23,10 @@ vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileByKey }, })) -import { readUserFileContent } from '@/lib/execution/payloads/materialization.server' +import { + readUserFileContent, + readUserFileContentWithContributors, +} from '@/lib/execution/payloads/materialization.server' import type { UserFile } from '@/executor/types' const PDF_SOURCE = Buffer.from('from reportlab.pdfgen import canvas') @@ -40,6 +44,7 @@ const generatedPdf: UserFile = { describe('readUserFileContent', () => { beforeEach(() => { vi.clearAllMocks() + resetDbChainMock() generatedPdf.size = PDF_SOURCE.length mockVerifyFileAccess.mockResolvedValue(true) mockReadWorkspaceFileByKey.mockResolvedValue({ file: { id: 'file-1' } }) @@ -49,6 +54,32 @@ describe('readUserFileContent', () => { }) }) + it('returns rendered contributor identities for the consuming boundary to classify', async () => { + const identity = { + fileId: 'image', + key: 'workspace/workspace-1/image.png', + context: 'workspace' as const, + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + } + const html = '' + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from(html), + contentType: 'text/html', + contributingFiles: [identity], + }) + + await expect( + readUserFileContentWithContributors( + { ...generatedPdf, name: 'page', type: 'text/x-sim-page' }, + { userId: 'user-1', encoding: 'text' } + ) + ).resolves.toEqual({ + content: html, + contributingFiles: [identity], + renderedContributingFiles: [identity], + }) + }) + it('returns the compiled artifact instead of the stored generation source', async () => { const content = await readUserFileContent(generatedPdf, { userId: 'user-1', diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index c5eafdedc6b..b919b3ca20b 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -63,6 +63,8 @@ export interface ReadUserFileContentOptions extends ExecutionMaterializationCont export interface ReadUserFileContentResult { content: string contributingFiles?: readonly WorkspaceFileSecretProvenanceIdentity[] + /** Subset transformed by the renderer; consumers apply their own admission policy. */ + renderedContributingFiles?: readonly WorkspaceFileSecretProvenanceIdentity[] } function getLogger(options: ExecutionMaterializationContext): Logger { @@ -215,10 +217,17 @@ function getExecutionKeyParts(key: string): } } +export class ExecutionFileAccessError extends Error { + constructor() { + super('File is not available in this execution.') + this.name = 'ExecutionFileAccessError' + } +} + function assertExecutionFileScope(key: string, options: ExecutionMaterializationContext): void { const parts = getExecutionKeyParts(key) if (!parts) { - throw new Error('File is not available in this execution.') + throw new ExecutionFileAccessError() } const allowedExecutionIds = new Set([ @@ -232,11 +241,11 @@ function assertExecutionFileScope(key: string, options: ExecutionMaterialization options.workflowId === parts.workflowId if (options.workspaceId && parts.workspaceId !== options.workspaceId) { - throw new Error('File is not available in this execution.') + throw new ExecutionFileAccessError() } if (options.workflowId && parts.workflowId !== options.workflowId) { - throw new Error('File is not available in this execution.') + throw new ExecutionFileAccessError() } if (allowedFileKeys.has(key)) { @@ -247,7 +256,7 @@ function assertExecutionFileScope(key: string, options: ExecutionMaterialization !options.executionId || (!allowedExecutionIds.has(parts.executionId) && !workflowScopeAllowed) ) { - throw new Error('File is not available in this execution.') + throw new ExecutionFileAccessError() } } @@ -344,7 +353,26 @@ export async function readUserFileContentWithContributors( throw new Error('Expected a file object with metadata.') } - await assertUserFileContentAccess(file, options) + let sourceIdentity: WorkspaceFileSecretProvenanceIdentity | undefined + const storageContext = file.key ? inferContextFromKey(file.key) : undefined + if ( + (storageContext === 'execution' || storageContext === 'workspace') && + options.principal && + options.workspaceId + ) { + const { resolveStoredFileProvenanceSource } = await import( + '@/lib/execution/payloads/file-secret-provenance' + ) + sourceIdentity = ( + await resolveStoredFileProvenanceSource(file, { + ...options, + principal: options.principal, + workspaceId: options.workspaceId, + }) + )?.identity + } else { + await assertUserFileContentAccess(file, options) + } const maxSourceBytes = options.maxSourceBytes ?? MAX_FUNCTION_FILE_BYTES if (Number.isFinite(file.size) && file.size > maxSourceBytes) { @@ -357,6 +385,7 @@ export async function readUserFileContentWithContributors( let buffer: Buffer | null = null let contributingFiles: readonly WorkspaceFileSecretProvenanceIdentity[] | undefined + let renderedContributingFiles: readonly WorkspaceFileSecretProvenanceIdentity[] | undefined const log = getLogger(options) const requestId = options.requestId ?? 'unknown' @@ -365,7 +394,10 @@ export async function readUserFileContentWithContributors( maxBytes: maxSourceBytes, }) buffer = servable.buffer - contributingFiles = servable.contributingFiles + renderedContributingFiles = servable.contributingFiles + contributingFiles = sourceIdentity + ? [sourceIdentity, ...(servable.contributingFiles ?? [])] + : servable.contributingFiles } catch (error) { if (isPayloadSizeLimitError(error)) { if (isGeneratedDocumentSourceType(file.type) && error.observedBytes !== undefined) { @@ -402,6 +434,7 @@ export async function readUserFileContentWithContributors( return { content: options.encoding === 'base64' ? bufferToBase64(selected) : selected.toString('utf8'), ...(contributingFiles && contributingFiles.length > 0 ? { contributingFiles } : {}), + ...(renderedContributingFiles?.length ? { renderedContributingFiles } : {}), } } diff --git a/apps/sim/lib/function-execution/application/execute-function.test.ts b/apps/sim/lib/function-execution/application/execute-function.test.ts index 156ee2d7542..4f4fa7542d0 100644 --- a/apps/sim/lib/function-execution/application/execute-function.test.ts +++ b/apps/sim/lib/function-execution/application/execute-function.test.ts @@ -26,6 +26,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization' import { executeFunction } from '@/lib/function-execution/application/execute-function' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const principal: WorkflowExecutionDelegatedPrincipal = { kind: 'delegated', @@ -154,4 +155,28 @@ describe('executeFunction', () => { expect(mocks.loadWorkspace).not.toHaveBeenCalled() expect(mocks.executeRequest).not.toHaveBeenCalled() }) + + it('passes trusted registry state outside the parsed Function wire body', async () => { + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'workspace-owner', + workspaceId: 'workspace-1', + }) + await executeFunction.execute({ + principal, + input: { + workspaceId: 'workspace-1', + body: { + code: 'return 1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + headers: new Headers(), + resolvedSecretTraceRegistry: registry, + }, + }) + + expect(mocks.executeRequest.mock.calls[0][2].resolvedSecretTraceRegistry).toBe(registry) + expect(mocks.executeRequest.mock.calls[0][1]).not.toHaveProperty('resolvedSecretTraceRegistry') + }) }) diff --git a/apps/sim/lib/function-execution/application/execute-function.ts b/apps/sim/lib/function-execution/application/execute-function.ts index 66b73058abe..c5a8e2998c4 100644 --- a/apps/sim/lib/function-execution/application/execute-function.ts +++ b/apps/sim/lib/function-execution/application/execute-function.ts @@ -5,6 +5,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { functionExecutionDelegationPolicy } from '@/lib/function-execution/application/authorization' import { functionExecutionOperations } from '@/lib/function-execution/application/operations' import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export interface ExecuteFunctionInput { workspaceId: string @@ -12,6 +13,8 @@ export interface ExecuteFunctionInput { headers: Headers signal?: AbortSignal sandboxProfile?: 'mothership' + /** Trusted in-process provenance state; never accepted from the Function request body. */ + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } /** @@ -57,6 +60,9 @@ export const executeFunction = defineAuthorizedWorkspaceUseCase({ { attributedUserId, principal, + ...(input.resolvedSecretTraceRegistry + ? { resolvedSecretTraceRegistry: input.resolvedSecretTraceRegistry } + : {}), ...(subject?.kind === 'sim_user' ? { fileAccessUserId: subject.userId } : {}), ...(input.sandboxProfile ? { sandboxProfile: input.sandboxProfile } : {}), } diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts index 4b821c084cb..36ceb0a0e59 100644 --- a/apps/sim/lib/function-execution/execute-request.test.ts +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -6,11 +6,14 @@ import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { createMockRequest, + dbChainMockFns, envFlagsMock, hybridAuthMockFns, + resetDbChainMock, resetEnvFlagsMock, workflowsUtilsMock, } from '@sim/testing' +import JSZip from 'jszip' import { NextRequest } from 'next/server' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import { functionExecuteBodySchema } from '@/lib/api/contracts' @@ -42,6 +45,9 @@ const { mockUploadFile, mockValidateWorkspaceFileWriteTarget, mockWriteWorkspaceFileByPath, + mockUploadExecutionFile, + mockMountContributors, + mockRenderedMountContributors, } = vi.hoisted(() => ({ mockExecuteInSandbox: vi.fn(), mockExecuteInIsolatedVM: vi.fn(), @@ -60,6 +66,9 @@ const { mockUploadFile: vi.fn(), mockValidateWorkspaceFileWriteTarget: vi.fn(), mockWriteWorkspaceFileByPath: vi.fn(), + mockUploadExecutionFile: vi.fn(), + mockMountContributors: vi.fn(), + mockRenderedMountContributors: vi.fn(), })) vi.mock('@/lib/core/security/encryption', () => ({ @@ -146,6 +155,10 @@ vi.mock('@/lib/uploads', () => ({ }, })) +vi.mock('@/lib/uploads/contexts/execution/execution-file-manager', () => ({ + uploadExecutionFile: mockUploadExecutionFile, +})) + vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) /** @@ -162,6 +175,8 @@ vi.mock('@/lib/function-execution/sandbox-mounts', () => ({ }: { planned: Array<{ userFile: { name: string }; mountPath: string }> }) => ({ + contributingFiles: mockMountContributors(), + renderedContributingFiles: mockRenderedMountContributors(), sandboxFiles: planned.map(({ mountPath }) => ({ type: 'url' as const, path: mountPath, @@ -180,9 +195,14 @@ import { validateExternalUrl } from '@/lib/core/security/input-validation' import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' +import * as fileMaterialization from '@/lib/execution/payloads/materialization.server' import { executeFunctionRequest } from '@/lib/function-execution/execute-request' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -async function POST(request: NextRequest): Promise { +async function POST( + request: NextRequest, + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry +): Promise { const auth = await hybridAuthMockFns.mockCheckInternalAuth(request) if (!auth.success || !auth.userId) { return Response.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) @@ -204,6 +224,7 @@ async function POST(request: NextRequest): Promise { return executeFunctionRequest({ headers: request.headers, signal: request.signal }, parsed.data, { attributedUserId: auth.userId, + resolvedSecretTraceRegistry, principal: { kind: 'delegated', serviceId: 'executor', @@ -247,6 +268,18 @@ const MOUNT_REF = { describe('Function execution request', () => { beforeEach(() => { vi.clearAllMocks() + resetDbChainMock() + mockMountContributors.mockReturnValue(undefined) + mockRenderedMountContributors.mockReturnValue(undefined) + mockUploadExecutionFile.mockImplementation(async (context, buffer, name, type) => ({ + id: 'execution-file-1', + key: `execution/${context.workspaceId}/${context.workflowId}/${context.executionId}/file/${name}`, + context: 'execution', + name, + type, + size: buffer.length, + url: 'https://presigned.example/output', + })) envFlagsMock.isRemoteSandboxEnabled = false envFlagsMock.isMothershipSandboxEnabled = false @@ -1792,6 +1825,375 @@ describe('Function execution request', () => { expect(data.error).toContain('21 files') }) + it.each([ + { name: 'report.zip', secret: undefined, expectedStatus: 'exact' }, + { name: 'report.zip', secret: 'super-secret-value', expectedStatus: 'unknown' }, + { name: 'report.txt', secret: 'super-secret-value', expectedStatus: 'unknown' }, + ])( + 'preserves binary provenance for harvested $name with secret=$secret', + async ({ name, secret, expectedStatus }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const zip = new JSZip() + zip.file('report.txt', secret ?? 'ordinary report') + const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) + expect(buffer.includes('super-secret-value')).toBe(false) + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: name, + path: `/tmp/sim/outputs/${name}`, + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + + const response = await POST( + createMockRequest('POST', { + code: secret ? 'token = {{MY_SECRET}}' : 'x = 1', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + ...(secret ? { envVars: { MY_SECRET: secret } } : {}), + }) + ) + + expect(response.status).toBe(200) + expect(mockUploadExecutionFile).toHaveBeenCalledWith( + expect.any(Object), + buffer, + name, + expect.any(String), + 'user-123', + expectedStatus === 'exact' ? { status: 'exact', entries: [] } : { status: 'unknown' } + ) + const data = await response.json() + expect(data.output.files[0]).not.toHaveProperty('secretProvenance') + } + ) + + it('keeps text-looking bytes opaque when their declared format is an archive', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const buffer = Buffer.from('ASCII archive placeholder') + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: 'report.zip', + path: '/tmp/sim/outputs/report.zip', + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + const response = await POST( + createMockRequest('POST', { + code: 'token = {{MY_SECRET}}', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + envVars: { MY_SECRET: 'super-secret-value' }, + }) + ) + expect(response.status).toBe(200) + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'unknown' }) + }) + + it('refuses an unknown tracked execution mount before running code', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const updatedAt = new Date('2026-01-01T00:00:00Z') + mockMountContributors.mockReturnValue([ + { + fileId: 'execution-file-1', + key: 'execution/workspace-1/workflow-1/execution-1/a/input.zip', + context: 'execution', + contentUpdatedAt: updatedAt, + }, + ]) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: updatedAt, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: updatedAt, + status: 'unknown', + entries: [], + }, + ]) + + const response = await POST( + createMockRequest('POST', { + code: 'x = 1', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + ) + expect(response.status).toBe(400) + expect((await response.json()).error).toContain('File secret provenance is unavailable') + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + }) + + it('imports exact mount secrets into the trusted result registry and binary export classifier', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const updatedAt = new Date('2026-01-01T00:00:00Z') + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-123', + workspaceId: 'workspace-1', + }) + const completePending = registry.beginPendingActivation() + mockMountContributors.mockReturnValue([ + { + fileId: 'execution-file-1', + key: 'execution/workspace-1/workflow-1/execution-1/a/input.txt', + context: 'execution', + contentUpdatedAt: updatedAt, + }, + ]) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: updatedAt, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: updatedAt, + status: 'exact', + entries: [ + { + name: 'API_KEY', + encryptedValue: 'encrypted:mounted-secret', + sourceUserId: 'user-123', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + ]) + const zip = new JSZip() + zip.file('result.txt', 'mounted-secret') + const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'mounted-secret', + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: 'result.zip', + path: '/tmp/sim/outputs/result.zip', + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + const response = await POST( + createMockRequest('POST', { + code: 'x = 1', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + registry + ) + completePending() + expect(response.status).toBe(200) + expect(registry.exportProvenance().entries).toEqual([ + expect.objectContaining({ encryptedValue: 'encrypted:mounted-secret' }), + ]) + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'unknown' }) + }) + + it.each([ + { input: 'contextVariables', archive: true, unredacted: false }, + { input: 'params', archive: true, unredacted: false }, + { input: 'contextVariables', archive: false, unredacted: false }, + { input: 'contextVariables', archive: true, unredacted: true }, + ] as const)( + 'classifies secret-bearing $input with archive=$archive and unredacted=$unredacted', + async ({ input, archive, unredacted }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const plaintext = 'table-input-secret-value' + const registry = new ResolvedSecretTraceRegistry( + [ + { + name: 'API_KEY', + plaintext, + encryptedValue: plaintext, + scope: 'workspace', + ...(unredacted ? { unredacted: true as const } : {}), + }, + ], + { userId: 'user-123', workspaceId: 'workspace-1' } + ) + registry.recordResolvedAtInputPath('API_KEY', plaintext, [input, 'token']) + const zip = new JSZip() + zip.file('report.txt', plaintext) + const buffer = archive + ? await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }) + : Buffer.from(plaintext) + const name = archive ? 'report.zip' : 'report.txt' + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: name, + path: `/tmp/sim/outputs/${name}`, + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + + const response = await POST( + createMockRequest('POST', { + code: input === 'params' ? "x = params['token']" : 'x = token', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + [input]: { token: plaintext }, + }), + registry + ) + + expect(response.status).toBe(archive || unredacted ? 200 : 400) + if (archive) { + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual( + unredacted ? { status: 'exact', entries: [] } : { status: 'unknown' } + ) + } else { + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + } + } + ) + + it.each([ + { reason: 'source-provenance-incomplete', status: 200 }, + { reason: 'entry-decrypt-failed', status: 400 }, + ] as const)( + 'distinguishes historical absence from provenance faults: $reason', + async ({ reason, status }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-123', + workspaceId: 'workspace-1', + }) + registry.markIncomplete(reason) + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + mockMountContributors.mockReturnValue([ + { + fileId: 'legacy-file', + key: 'execution/workspace-1/workflow-1/execution-1/a/input.txt', + context: 'execution', + contentUpdatedAt, + }, + ]) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: contentUpdatedAt, + secretProvenanceVersion: null, + provenanceContentUpdatedAt: null, + status: null, + entries: null, + }, + ]) + const buffer = Buffer.from('ordinary file') + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: 'report.zip', + path: '/tmp/sim/outputs/report.zip', + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + const response = await POST( + createMockRequest('POST', { + code: 'x = 1', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + registry + ) + expect(response.status).toBe(status) + if (status === 200) + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'unrecorded' }) + else expect(mockUploadExecutionFile).not.toHaveBeenCalled() + } + ) + + it('does not taint a secret-free mounted file with unrelated secrets from an earlier block', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const updatedAt = new Date('2026-01-01T00:00:00Z') + const scope = { userId: 'user-123', workspaceId: 'workspace-1' } + const registry = new ResolvedSecretTraceRegistry([], scope) + await registry.importProvenance( + { + version: 1, + complete: true, + scope, + entries: [{ name: 'OTHER_SECRET', encryptedValue: 'unrelated-secret-value' }], + }, + { trusted: true } + ) + mockMountContributors.mockReturnValue([ + { + fileId: 'execution-file-1', + key: 'execution/workspace-1/workflow-1/execution-1/a/input.txt', + context: 'execution', + contentUpdatedAt: updatedAt, + }, + ]) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: updatedAt, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: updatedAt, + status: 'exact', + entries: [], + }, + ]) + const buffer = Buffer.from('archive without secret inputs') + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: '', + sandboxId: 'sbx', + collectedFiles: [ + { + relativePath: 'result.zip', + path: '/tmp/sim/outputs/result.zip', + contentBase64: buffer.toString('base64'), + byteLength: buffer.length, + }, + ], + }) + const response = await POST( + createMockRequest('POST', { + code: 'x = 1', + language: 'python', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + registry + ) + expect(response.status).toBe(200) + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'exact', entries: [] }) + }) + it('scans a harvested plaintext secret even under a binary file name', async () => { envFlagsMock.isRemoteSandboxEnabled = true mockExecuteInSandbox.mockResolvedValueOnce({ @@ -2479,6 +2881,112 @@ describe('Function execution request', () => { expect(data.success).toBe(true) expect(options?.brokers).toHaveProperty('sim.values.readArray') }) + + it.each([ + { status: 'exact', version: 1, secret: true, safe: false }, + { status: 'unknown', version: 1, secret: false, safe: false }, + { status: 'exact', version: 1, secret: false, safe: true }, + { status: null, version: null, secret: false, safe: true }, + ])( + 'applies rendered asset policy at Function admission while retaining legacy compatibility: %j', + async ({ status, version, secret, safe }) => { + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + const identity = { + fileId: 'image-1', + key: 'workspace/workspace-1/image.png', + context: 'workspace' as const, + contentUpdatedAt, + } + const materialized = { + content: '', + contributingFiles: [identity], + renderedContributingFiles: [identity], + } + const read = vi + .spyOn(fileMaterialization, 'readUserFileContentWithContributors') + .mockResolvedValue(materialized) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: contentUpdatedAt, + provenanceContentUpdatedAt: contentUpdatedAt, + secretProvenanceVersion: version, + status, + entries: secret + ? [{ name: 'TOKEN', encryptedValue: 'ciphertext', sourceUserId: 'user-1' }] + : [], + }, + ]) + mockExecuteInIsolatedVM.mockImplementationOnce(async (_input, options) => ({ + result: await options.brokers['sim.files.readText']({ file: MOUNT_REF.file }), + stdout: '', + })) + const request = { + code: 'return 1', + language: 'javascript', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + try { + const brokerResponse = await POST(createMockRequest('POST', request)) + const brokerBody = await brokerResponse.json() + expect(brokerBody.success).toBe(safe) + if (!safe) expect(JSON.stringify(brokerBody)).not.toContain(materialized.content) + + mockMountContributors.mockReturnValue([identity]) + mockRenderedMountContributors.mockReturnValue([identity]) + const mountResponse = await POST(createMockRequest('POST', request)) + expect(mountResponse.status).toBe(safe ? 200 : 400) + expect((await mountResponse.json()).success).toBe(safe) + } finally { + read.mockRestore() + } + } + ) + + it('refuses unknown execution provenance returned by the runtime file broker', async () => { + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + vi.spyOn(fileMaterialization, 'readUserFileContentWithContributors').mockResolvedValueOnce({ + content: 'private file content', + contributingFiles: [ + { + fileId: 'execution-file-1', + key: 'execution/workspace-1/workflow-1/execution-1/a/input.txt', + context: 'execution', + contentUpdatedAt, + }, + ], + }) + dbChainMockFns.limit.mockResolvedValue([ + { + fileContentUpdatedAt: contentUpdatedAt, + secretProvenanceVersion: 1, + provenanceContentUpdatedAt: contentUpdatedAt, + status: 'unknown', + entries: [], + }, + ]) + mockExecuteInIsolatedVM.mockImplementationOnce(async (_input, options) => ({ + result: await options.brokers['sim.files.readText']({ file: MOUNT_REF.file }), + stdout: '', + })) + + const response = await POST( + createMockRequest('POST', { + code: 'return 1', + language: 'javascript', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + ) + + expect(response.status).toBe(500) + const data = await response.json() + expect(data.success).toBe(false) + expect(data.error).toContain('File secret provenance is unavailable') + expect(JSON.stringify(data)).not.toContain('private file content') + }) }) describe('Template Variable Resolution', () => { diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index 1392221fbd0..020b7b32f9d 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -55,7 +55,7 @@ import { MAX_INLINE_MATERIALIZATION_BYTES, } from '@/lib/execution/payloads/limits' import { - readUserFileContent, + readUserFileContentWithContributors, unavailableLargeValueError, } from '@/lib/execution/payloads/materialization.server' import { @@ -93,9 +93,13 @@ import { isExecutionResourceLimitError } from '@/lib/execution/resource-errors' import { planUserFileMounts, resolveUserFileMounts } from '@/lib/function-execution/sandbox-mounts' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' import { + createWorkspaceFileSecretProvenanceFromRegistry, EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + importWorkspaceFileSecretProvenanceForRuntime, + isOpaqueWorkspaceFileEgressSafe, mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenanceIdentity, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { deleteFiles } from '@/lib/uploads/core/storage-service' import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' @@ -117,7 +121,12 @@ import { scanResolvedSecretString, } from '@/executor/utils/resolved-secret-content-projection' import { isNonIdentifyingSecretLiteral } from '@/executor/utils/resolved-secret-match-policy' -import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' +import type { + ResolvedSecretTraceProvenanceV1, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +const TEXT_OUTPUT_MIME_TYPES = new Set(Object.values(FORMAT_TO_CONTENT_TYPE)) const logger = createLogger('FunctionExecuteAPI') @@ -1014,6 +1023,74 @@ interface FunctionRouteExecutionContext { */ unredactedSecretNames: Set mountedFileSecretProvenanceScanner?: MountedFileSecretProvenanceScanner + runtimeFileSecretProvenanceScanner?: MountedFileSecretProvenanceScanner + runtimeFileSecretTraceRegistry?: ResolvedSecretTraceRegistry + runtimeInputProvenanceUnrecorded?: boolean + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry +} + +/** Keeps bound file provenance in both ordinary Function results and exported artifact bytes. */ +async function importRuntimeFileContributors( + context: FunctionRouteExecutionContext, + identities: readonly WorkspaceFileSecretProvenanceIdentity[] | undefined, + renderedIdentities: readonly WorkspaceFileSecretProvenanceIdentity[] = [] +): Promise { + if (!identities?.length && renderedIdentities.length === 0) return + if (!context.workspaceId) throw new Error('File provenance requires a workspace') + /** Sim-rendered assets can encode literals before user code runs; raw files retain runtime lineage. */ + for (const identity of renderedIdentities) { + if (!(await isOpaqueWorkspaceFileEgressSafe(context.workspaceId, identity))) { + throw new Error('File secret provenance is unavailable for Function execution') + } + } + if (!context.runtimeInputProvenanceUnrecorded) { + context.runtimeFileSecretTraceRegistry ??= + context.resolvedSecretTraceRegistry?.forkForInputPaths([]) + } + for (const identity of identities ?? []) { + const imported = await importWorkspaceFileSecretProvenanceForRuntime({ + workspaceId: context.workspaceId, + identity, + registry: context.runtimeFileSecretTraceRegistry, + actorUserId: context.fileAccessUserId, + }) + if (!imported) throw new Error('File secret provenance is unavailable for Function execution') + } + if (context.runtimeFileSecretTraceRegistry && context.resolvedSecretTraceRegistry) { + context.resolvedSecretTraceRegistry.mergeToolCallRegistry( + context.runtimeFileSecretTraceRegistry + ) + } +} + +/** Includes only lineage carried by the values this Function receives, including deferred refs. */ +async function importRuntimeInputProvenance( + context: FunctionRouteExecutionContext, + inputs: { + code: string + params: Record + contextVariables: Record + } +): Promise { + const registry = context.resolvedSecretTraceRegistry + if (!registry) return + const valueProvenance = registry.exportCommittedProvenanceForValue(inputs) + if (!valueProvenance.complete && context.workspaceId) { + const decision = await createWorkspaceFileSecretProvenanceFromRegistry(registry, inputs, { + userId: context.attributedUserId, + workspaceId: context.workspaceId, + }) + if (decision.safe && decision.provenance.status === 'unrecorded') { + context.runtimeInputProvenanceUnrecorded = true + return + } + } + const inputRegistry = registry.forkForInputPaths(Object.keys(inputs).map((key) => [key])) + await inputRegistry.importProvenance(valueProvenance, { + trusted: true, + origin: 'function.runtimeInputs', + }) + context.runtimeFileSecretTraceRegistry = inputRegistry } type ResolvedSecretNamesMetadataType = @@ -1117,7 +1194,7 @@ function createFunctionRuntimeBrokers( const readFile = async (args: unknown, encoding: 'base64' | 'text', chunked = false) => { const fileArgs = getBrokerFileArgs(args) - return readUserFileContent(fileArgs.file, { + const materialized = await readUserFileContentWithContributors(fileArgs.file, { ...base, encoding, maxBytes: fileArgs.maxBytes, @@ -1125,6 +1202,12 @@ function createFunctionRuntimeBrokers( offset: chunked ? fileArgs.offset : undefined, length: chunked ? fileArgs.length : undefined, }) + await importRuntimeFileContributors( + context, + materialized.contributingFiles, + materialized.renderedContributingFiles + ) + return materialized.content } return { @@ -1256,7 +1339,10 @@ function countProtectedOutputSecretNames(context: FunctionRouteExecutionContext) */ function hasSecretMaterialInScope(context: FunctionRouteExecutionContext): boolean { if (countProtectedOutputSecretNames(context) > 0) return true - return context.mountedFileSecretProvenanceScanner?.hasSecrets ?? false + return Boolean( + context.mountedFileSecretProvenanceScanner?.hasSecrets || + context.runtimeFileSecretProvenanceScanner?.hasSecrets + ) } /** @@ -1274,15 +1360,34 @@ async function getOutputFileSecretProvenance( context: FunctionRouteExecutionContext, scope: { userId: string; workspaceId: string } ): Promise { + /** Runtime reads have settled before export; a broker cannot replace this with an older snapshot. */ + if (context.runtimeFileSecretTraceRegistry && !context.runtimeFileSecretProvenanceScanner) { + const provenance = context.runtimeFileSecretTraceRegistry.exportProvenance() + context.runtimeFileSecretProvenanceScanner = + await createMountedFileSecretProvenanceScanner(provenance) + if (!context.runtimeFileSecretProvenanceScanner && provenance.entries.length > 0) { + context.runtimeFileSecretProvenanceScanner = { + hasSecrets: true, + scan: () => ({ status: 'unknown' }), + } + } + } if (isBinary) { return hasSecretMaterialInScope(context) ? { status: 'unknown' } - : EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE - } - const mountedFileProvenance = context.mountedFileSecretProvenanceScanner?.scan(buffer) ?? { - status: 'exact' as const, - entries: [], + : context.runtimeInputProvenanceUnrecorded + ? { status: 'unrecorded' } + : EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE } + const mountedFileProvenance = mergeWorkspaceFileSecretProvenance( + context.mountedFileSecretProvenanceScanner?.scan(buffer) ?? + EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + context.runtimeFileSecretProvenanceScanner?.scan(buffer) ?? + EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE, + context.runtimeInputProvenanceUnrecorded + ? { status: 'unrecorded' } + : EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE + ) if (countProtectedOutputSecretNames(context) === 0) { return mountedFileProvenance } @@ -1531,12 +1636,11 @@ async function maybeExportSandboxFileToWorkspace(args: { const fileName = normalizeOutputWorkspaceFileName(outputPath) - const TEXT_MIMES = new Set(Object.values(FORMAT_TO_CONTENT_TYPE)) const resolvedMimeType = outputMimeType || FORMAT_TO_CONTENT_TYPE[resolveOutputFormat(fileName, outputFormat)] || 'application/octet-stream' - const isBinary = !TEXT_MIMES.has(resolvedMimeType) + const isBinary = !TEXT_OUTPUT_MIME_TYPES.has(resolvedMimeType) const outputBytes = Buffer.byteLength(exportedFileContent, isBinary ? 'base64' : 'utf-8') if (outputBytes > MAX_SANDBOX_OUTPUT_BYTES) { return exportFailure( @@ -1711,7 +1815,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { file.mimeType || FORMAT_TO_CONTENT_TYPE[resolveOutputFormat(fileName, file.format)] || 'application/octet-stream' - const isBinary = !new Set(Object.values(FORMAT_TO_CONTENT_TYPE)).has(resolvedMimeType) + const isBinary = !TEXT_OUTPUT_MIME_TYPES.has(resolvedMimeType) const size = Buffer.byteLength(content, isBinary ? 'base64' : 'utf-8') totalOutputBytes += size if (totalOutputBytes > MAX_SANDBOX_OUTPUT_BYTES) { @@ -1926,16 +2030,6 @@ function collectedFileName(relativePath: string): string { return sanitizeFileName(relativePath.split('/').filter(Boolean).join('-')) || 'file' } -/** - * Persists files harvested from the sandbox output directory as platform file - * objects, so any downstream tool that accepts a file can consume them. - * - * Uploaded here, one at a time, rather than handed to the declarative - * file-output pipeline as bytes: that path would carry the whole export budget - * as base64 through `JSON.stringify`, a response buffer, and a re-parse, so - * several multiples of the payload would be live at once for a value that is a - * couple of hundred bytes per file once stored. - */ /** * Removes files already uploaded when a later one in the same harvest is refused. * @@ -1959,6 +2053,7 @@ async function discardUploadedExecutionFiles(files: readonly UserFile[]): Promis } } +/** Uploads harvested files sequentially, retaining their private provenance beside stored bytes. */ async function collectExecutionOutputFiles(args: { routeContext: FunctionRouteExecutionContext authUserId: string @@ -2001,38 +2096,41 @@ async function collectExecutionOutputFiles(args: { const name = collectedFileName(collected.relativePath) const mimeType = getMimeTypeFromExtension(getFileExtension(name)) - // Scanned unconditionally — never gated on whether the bytes look textual. - // Both a filename check and a UTF-8 round-trip were trivially defeated: name - // the file `.png`, or append one invalid byte, and a plaintext secret sailed - // past. A lossy UTF-8 decode preserves ASCII runs, so a literal secret is - // findable in any buffer, textual or not. - // - // What stays out of reach is a secret carried in transformed form — deflated - // inside a PDF, re-encoded — which no substring scan can see. That is an - // inherent limit of scanning, not a hole in the gate, and it is why these - // files are execution-scoped rather than durable workspace files. - { - const provenance = await getOutputFileSecretProvenance(buffer, false, routeContext, { - userId: args.authUserId, - workspaceId: resolvedWorkspaceId, - }) - // An execution-scoped file has nowhere to record a provenance envelope, so - // one carrying a resolved secret cannot ship under a lock the way a - // workspace file can — it is refused instead. - if (provenance.status !== 'exact' || provenance.entries.length > 0) { - await discardUploadedExecutionFiles(files) - return { - response: exportFailure( - `Sandbox output file "${name}" contains a resolved secret value and was not returned. Write the file without embedding secret values, or export it to a workspace file where its provenance can be recorded.`, - 400, - args.stdout, - args.executionTime, - args.cost - ), - } + /** Literal secrets must be refused regardless of the export's name or encoding. */ + const scannedProvenance = await getOutputFileSecretProvenance(buffer, false, routeContext, { + userId: args.authUserId, + workspaceId: resolvedWorkspaceId, + }) + if ( + scannedProvenance.status === 'unknown' || + (scannedProvenance.status === 'exact' && scannedProvenance.entries.length > 0) + ) { + await discardUploadedExecutionFiles(files) + return { + response: exportFailure( + `Sandbox output file "${name}" contains a resolved secret value and was not returned. Write the file without embedding secret values, or export it to a workspace file where its provenance can be recorded.`, + 400, + args.stdout, + args.executionTime, + args.cost + ), } } + /** + * A literal scan cannot vouch for encoded secrets in an archive or binary document. + * Persist that uncertainty so a later conversion cannot turn these bytes into a trusted + * workspace file. Both the format and bytes must be textual before a scan is sufficient. + */ + const isBinary = + !TEXT_OUTPUT_MIME_TYPES.has(mimeType) || !isUtf8(buffer) || buffer.includes(0) + const secretProvenance = isBinary + ? await getOutputFileSecretProvenance(buffer, true, routeContext, { + userId: args.authUserId, + workspaceId: resolvedWorkspaceId, + }) + : scannedProvenance + const userFile = await uploadExecutionFile( { workspaceId: resolvedWorkspaceId, @@ -2042,7 +2140,8 @@ async function collectExecutionOutputFiles(args: { buffer, name, mimeType, - args.authUserId + args.authUserId, + secretProvenance ) files.push(userFile) } @@ -2070,6 +2169,7 @@ export interface TrustedFunctionExecutionAuth { fileAccessUserId?: string principal: DelegatedPrincipal sandboxProfile?: 'mothership' + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } /** Executes the Function protocol after the application operation authorizes its principal. */ @@ -2284,6 +2384,7 @@ export async function executeFunctionRequest( unredactedSecretNames.filter((name) => Object.hasOwn(envVars, name)) ), mountedFileSecretProvenanceScanner, + resolvedSecretTraceRegistry: auth.resolvedSecretTraceRegistry, } const lang = isValidCodeLanguage(language) ? language : DEFAULT_CODE_LANGUAGE @@ -2377,6 +2478,11 @@ export async function executeFunctionRequest( for (const binding of compilation.bindings) { setRecordValue(contextVariables, binding.name, binding.value) } + await importRuntimeInputProvenance(routeContext, { + code: resolvedCode, + params: executionParams, + contextVariables, + }) if (lang === CodeLanguage.Shell && containsLargeValueRef(contextVariables)) { throw new Error( 'Large execution values require the JavaScript isolated-vm runtime. Select a nested field or read the value in a JavaScript function.' @@ -2479,6 +2585,11 @@ export async function executeFunctionRequest( logger, }, }) + await importRuntimeFileContributors( + routeContext, + resolvedMounts.contributingFiles, + resolvedMounts.renderedContributingFiles + ) } catch (error) { // Everything this can raise is about the files the caller named — a mount // it may not read, one over a size ceiling, a set over the aggregate. The @@ -3258,3 +3369,5 @@ export async function executeFunctionRequest( executionDeadlineController?.cleanup() } } + +import { isUtf8 } from 'node:buffer' diff --git a/apps/sim/lib/function-execution/sandbox-mounts.test.ts b/apps/sim/lib/function-execution/sandbox-mounts.test.ts index 2f49c989885..6557a35cb0d 100644 --- a/apps/sim/lib/function-execution/sandbox-mounts.test.ts +++ b/apps/sim/lib/function-execution/sandbox-mounts.test.ts @@ -14,11 +14,13 @@ const { mockGeneratePresignedDownloadUrl, mockDownloadServableFileFromStorage, mockReadWorkspaceFileRecordByKey, + mockGetFileMetadataByKey, } = vi.hoisted(() => ({ mockHasCloudStorage: vi.fn(), mockGeneratePresignedDownloadUrl: vi.fn(), mockDownloadServableFileFromStorage: vi.fn(), mockReadWorkspaceFileRecordByKey: vi.fn(), + mockGetFileMetadataByKey: vi.fn(), })) vi.mock('@/lib/uploads/core/storage-service', () => ({ @@ -34,6 +36,10 @@ vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileRecordByKey }, })) +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataByKey: mockGetFileMetadataByKey, +})) + import { MOUNT_URL_TTL_SECONDS, planUserFileMounts, @@ -137,6 +143,7 @@ describe('resolveUserFileMounts', () => { mockHasCloudStorage.mockReturnValue(true) mockGeneratePresignedDownloadUrl.mockResolvedValue('https://presigned.example/object') mockReadWorkspaceFileRecordByKey.mockResolvedValue({ file: { id: 'wf_1' } }) + mockGetFileMetadataByKey.mockResolvedValue(null) // Sized from the file being read: the aggregate budget counts bytes actually // buffered, so a fixed-size stub would never let the total ceiling trip. mockDownloadServableFileFromStorage.mockImplementation(async (file: UserFile) => ({ @@ -175,6 +182,88 @@ describe('resolveUserFileMounts', () => { ]) }) + it('carries canonical execution provenance through a URL mount without buffering bytes', async () => { + const file = executionFile({ id: 'untrusted-public-id' }) + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + mockGetFileMetadataByKey.mockResolvedValue({ + id: 'canonical-file-id', + key: file.key, + context: 'execution', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + contentUpdatedAt, + }) + + const result = await resolveUserFileMounts({ + planned: planUserFileMounts([file]), + context: { + ...executionContext, + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + }, + }) + + expect(result.contributingFiles).toEqual([ + { + fileId: 'canonical-file-id', + key: file.key, + context: 'execution', + contentUpdatedAt, + }, + ]) + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('preserves contributors introduced when an inline mount renders generated source', async () => { + const contributor = { + fileId: 'image-file', + key: 'workspace/ws-1/image.png', + context: 'workspace' as const, + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + } + mockHasCloudStorage.mockReturnValue(false) + mockDownloadServableFileFromStorage.mockResolvedValueOnce({ + buffer: Buffer.from('rendered'), + contributingFiles: [contributor], + }) + const result = await resolveUserFileMounts({ + planned: planUserFileMounts([executionFile()]), + context: executionContext, + }) + expect(result.contributingFiles).toEqual([contributor]) + expect(result.renderedContributingFiles).toEqual([contributor]) + }) + + it('retains both revisions when a file changes between two mount resolutions', async () => { + const oldFile = workspaceFile({ key: 'workspace/ws-1/old.pdf' }) + const newFile = workspaceFile({ key: 'workspace/ws-1/new.pdf' }) + const revisions = [new Date('2026-01-01T00:00:00Z'), new Date('2026-01-01T00:01:00Z')] + for (const [index, file] of [oldFile, newFile].entries()) { + mockGetFileMetadataByKey.mockResolvedValueOnce({ + id: 'canonical-file-id', + key: file.key, + context: 'workspace', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + contentUpdatedAt: revisions[index], + }) + } + const result = await resolveUserFileMounts({ + planned: planUserFileMounts([oldFile, newFile]), + context: { + ...executionContext, + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + }, + }) + expect(result.contributingFiles).toEqual( + [oldFile, newFile].map((file, index) => ({ + fileId: 'canonical-file-id', + key: file.key, + context: 'workspace', + contentUpdatedAt: revisions[index], + })) + ) + }) + it('buffers bytes inline when there is no cloud storage to presign from', async () => { mockHasCloudStorage.mockReturnValue(false) diff --git a/apps/sim/lib/function-execution/sandbox-mounts.ts b/apps/sim/lib/function-execution/sandbox-mounts.ts index cbad1fa2859..8ba9069dcef 100644 --- a/apps/sim/lib/function-execution/sandbox-mounts.ts +++ b/apps/sim/lib/function-execution/sandbox-mounts.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { resolveStoredFileProvenanceSource } from '@/lib/execution/payloads/file-secret-provenance' import { assertUserFileContentAccess, type ExecutionMaterializationContext, @@ -7,6 +8,7 @@ import { import { MAX_SANDBOX_URL_MOUNT_BYTES } from '@/lib/execution/remote-sandbox/output-limits' import { SANDBOX_INPUT_DIR } from '@/lib/execution/remote-sandbox/sandbox-paths' import type { SandboxFile } from '@/lib/execution/remote-sandbox/types' +import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' import { generatePresignedDownloadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' import type { StorageContext } from '@/lib/uploads/shared/types' @@ -270,14 +272,39 @@ export function planUserFileMounts( export async function resolveUserFileMounts(args: { planned: readonly PlannedUserFileMount[] context: ExecutionMaterializationContext -}): Promise<{ sandboxFiles: SandboxFile[]; manifest: SandboxMountManifestEntry[] }> { +}): Promise<{ + sandboxFiles: SandboxFile[] + manifest: SandboxMountManifestEntry[] + contributingFiles?: readonly WorkspaceFileSecretProvenanceIdentity[] + renderedContributingFiles?: readonly WorkspaceFileSecretProvenanceIdentity[] +}> { const sandboxFiles: SandboxFile[] = [] const manifest: SandboxMountManifestEntry[] = [] const budget = createSandboxMountBudget() + const contributingFiles = new Map() + const renderedContributingFiles = new Map() + const addContributor = (identity: WorkspaceFileSecretProvenanceIdentity, rendered = false) => { + const revision = JSON.stringify([ + identity.fileId, + identity.key, + identity.context, + identity.contentUpdatedAt?.getTime(), + ]) + contributingFiles.set(revision, identity) + if (rendered) renderedContributingFiles.set(revision, identity) + } for (const { userFile, mountPath } of args.planned) { const storageContext = resolveTrustedFileContext(userFile.key, userFile.context) await assertUserFileContentAccess(userFile, args.context) + if (args.context.principal && args.context.workspaceId) { + const source = await resolveStoredFileProvenanceSource(userFile, { + ...args.context, + principal: args.context.principal, + workspaceId: args.context.workspaceId, + }) + if (source) addContributor(source.identity) + } await pushSandboxFileMount( sandboxFiles, @@ -291,12 +318,22 @@ export async function resolveUserFileMounts(args: { // Base64 regardless of content type: the payload is reproduced exactly // for any byte sequence, and picking utf8 for a mistyped binary would // substitute U+FFFD and hand the code a corrupted file. - const { content } = await readUserFileContentWithContributors(userFile, { + const { + content, + contributingFiles: contributors, + renderedContributingFiles, + } = await readUserFileContentWithContributors(userFile, { ...args.context, encoding: 'base64', maxBytes, maxSourceBytes: maxBytes, }) + for (const contributor of contributors ?? []) { + addContributor(contributor) + } + for (const contributor of renderedContributingFiles ?? []) { + addContributor(contributor, true) + } return { content, encoding: 'base64' as const, @@ -321,5 +358,12 @@ export async function resolveUserFileMounts(args: { urlBytes: budget.url, }) - return { sandboxFiles, manifest } + return { + sandboxFiles, + manifest, + ...(contributingFiles.size > 0 ? { contributingFiles: [...contributingFiles.values()] } : {}), + ...(renderedContributingFiles.size > 0 + ? { renderedContributingFiles: [...renderedContributingFiles.values()] } + : {}), + } } diff --git a/apps/sim/lib/internal/file/execute-tool.test.ts b/apps/sim/lib/internal/file/execute-tool.test.ts index 82853d417ac..05534d68ab1 100644 --- a/apps/sim/lib/internal/file/execute-tool.test.ts +++ b/apps/sim/lib/internal/file/execute-tool.test.ts @@ -298,8 +298,11 @@ describe('executeFileTool', () => { }) it.each(PARSER_TOOL_IDS)('dispatches %s with trusted execution scope', async (toolId) => { + const headers = new Headers({ + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', + }) const response = await executeFileTool( - request(toolId, { filePath: 'https://example.com/report.txt', fileType: '' }) + request(toolId, { filePath: 'https://example.com/report.txt', fileType: '' }, { headers }) ) expect(response.status).toBe(200) @@ -311,6 +314,7 @@ describe('executeFileTool', () => { executionId: 'execution-1', attributedUserId: 'user-1', fileAccessUserId: 'user-1', + headers, }) ) expect(mocks.executeManage).not.toHaveBeenCalled() diff --git a/apps/sim/lib/internal/file/execute-tool.ts b/apps/sim/lib/internal/file/execute-tool.ts index 4ed4dd0a2d5..5641a165781 100644 --- a/apps/sim/lib/internal/file/execute-tool.ts +++ b/apps/sim/lib/internal/file/execute-tool.ts @@ -166,6 +166,7 @@ export const executeFileTool: InternalToolOperationHandler = async (request) => fileKeys: request.context.fileKeys, allowLargeValueWorkflowScope: request.context.allowLargeValueWorkflowScope, requestId: request.requestId, + headers: request.headers, signal: request.signal, }) } else { diff --git a/apps/sim/lib/internal/file/operations.provenance.test.ts b/apps/sim/lib/internal/file/operations.provenance.test.ts index 9e129d0f1bd..e82ebb64904 100644 --- a/apps/sim/lib/internal/file/operations.provenance.test.ts +++ b/apps/sim/lib/internal/file/operations.provenance.test.ts @@ -177,7 +177,10 @@ vi.mock('@/lib/core/security/encryption', () => ({ import { fileManageBodySchema } from '@/lib/api/contracts/tools/file' import type { DbTransaction } from '@/lib/db/types' -import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { + executeFileManageOperation, + getFileContentProvenance, +} from '@/lib/internal/file/operations' import { importWorkspaceFileSecretProvenanceForModelView, isOpaqueWorkspaceFileEgressSafe, @@ -395,3 +398,102 @@ describe('appended file provenance', () => { } ) }) + +describe('execution-file content provenance', () => { + const identity = { + fileId: 'execution-file', + key: 'execution/workspace-1/workflow-1/execution-1/report.txt', + context: 'execution' as const, + contentUpdatedAt: CONTENT_UPDATED_AT, + } + const principal = createWorkspaceFileDelegatedPrincipal({ + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'test-file-content', + }) + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it.each([ + { status: 'exact', version: 1, stale: false, enforced: false, complete: true }, + { status: 'exact', version: 1, stale: false, enforced: true, complete: true }, + { status: 'unrecorded', version: 1, stale: false, enforced: false, complete: true }, + { status: 'unrecorded', version: 1, stale: false, enforced: true, complete: false }, + { status: 'unknown', version: 1, stale: false, enforced: false, complete: false }, + { status: 'unknown', version: 1, stale: false, enforced: true, complete: false }, + { status: 'unknown', version: null, stale: false, enforced: false, complete: true }, + { status: 'unknown', version: null, stale: false, enforced: true, complete: true }, + { status: 'exact', version: 1, stale: true, enforced: false, complete: false }, + { status: 'exact', version: 1, stale: true, enforced: true, complete: false }, + ])( + 'reads $status version=$version stale=$stale with enforcement=$enforced', + async ({ status, version, stale, enforced, complete }) => { + mockEnforced.mockReturnValue(enforced) + queueTableRows(workspaceFiles, [ + { + ...joinedRow(status), + secretProvenanceVersion: version, + ...(stale ? { provenanceContentUpdatedAt: new Date(0) } : {}), + }, + ]) + + const provenance = await getFileContentProvenance(principal, 'workspace-1', [ + { identity, ownerUserId: 'user-1' }, + ]) + + expect(provenance).toMatchObject({ version: 1, complete, entries: [] }) + } + ) + + it('retains exact secret-bearing execution lineage for downstream text projections', async () => { + mockEnforced.mockReturnValue(true) + queueTableRows(workspaceFiles, [ + joinedRow('exact', [ + { + name: 'TOKEN', + encryptedValue: 'synthetic-ciphertext', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ]), + ]) + + const provenance = await getFileContentProvenance(principal, 'workspace-1', [ + { identity, ownerUserId: 'user-1' }, + ]) + const registry = new ResolvedSecretTraceRegistry([], SCOPE) + expect(provenance.complete).toBe(true) + expect(await registry.importProvenance(provenance, { trusted: true })).toBe(true) + expect(projectResolvedSecretModelContent(`parsed: ${SECRET}`, registry)).toEqual({ + safe: true, + value: 'parsed: {{TOKEN}}', + }) + }) + + it.each([ + { sourceUserId: 'other-user', sourceWorkspaceId: 'workspace-1' }, + { sourceUserId: 'user-1', sourceWorkspaceId: 'other-workspace' }, + ])('anonymizes names from a different source scope: %j', async (sourceScope) => { + queueTableRows(workspaceFiles, [ + joinedRow('exact', [ + { name: 'PRIVATE_SOURCE_NAME', encryptedValue: 'synthetic-ciphertext', ...sourceScope }, + ]), + ]) + + const provenance = await getFileContentProvenance(principal, 'workspace-1', [ + { identity, ownerUserId: 'user-1' }, + ]) + + expect(provenance).toEqual({ + version: 1, + complete: true, + entries: [{ encryptedValue: 'synthetic-ciphertext' }], + scope: SCOPE, + }) + expect(JSON.stringify(provenance)).not.toContain('PRIVATE_SOURCE_NAME') + }) +}) diff --git a/apps/sim/lib/internal/file/operations.test.ts b/apps/sim/lib/internal/file/operations.test.ts index 2516a1873a7..3cffe984456 100644 --- a/apps/sim/lib/internal/file/operations.test.ts +++ b/apps/sim/lib/internal/file/operations.test.ts @@ -128,6 +128,8 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ getWorkspaceFileByName: (...args: unknown[]) => mockGetWorkspaceFileByName(...args), getWorkspaceFile: (...args: unknown[]) => mockGetWorkspaceFile(...args), loadActiveWorkspaceContext: (...args: unknown[]) => mockLoadActiveWorkspaceContext(...args), + loadActiveWorkspaceFileContext: (...args: unknown[]) => + mockLoadActiveWorkspaceFileContext(...args), updateWorkspaceFileContent: (...args: unknown[]) => mockUpdateWorkspaceFileContent(...args), uploadWorkspaceFile: (...args: unknown[]) => mockUploadWorkspaceFile(...args), })) @@ -1068,13 +1070,30 @@ describe('file manage operations', () => { ? { status: 'exact', entries: [ - { name: 'TOKEN', encryptedValue: 'encrypted-token' }, - { name: 'ALPHA', encryptedValue: 'encrypted-alpha' }, + { + name: 'TOKEN', + encryptedValue: 'encrypted-token', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + { + name: 'ALPHA', + encryptedValue: 'encrypted-alpha', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, ], } : { status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + entries: [ + { + name: 'TOKEN', + encryptedValue: 'encrypted-token', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ], } ) @@ -1115,6 +1134,215 @@ describe('file manage operations', () => { ) }) + describe('rendered file contributors', () => { + const contributor = { + fileId: 'image', + key: 'workspace/workspace-1/image.txt', + context: 'workspace' as const, + contentUpdatedAt: CONTENT_UPDATED_AT, + } + const secretEntries = [ + { + name: 'IMAGE_TOKEN', + encryptedValue: 'encrypted-image-token', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ] + + beforeEach(() => { + mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('document')) + mockGetFileMetadataByKey.mockResolvedValue({ + id: contributor.fileId, + key: contributor.key, + context: contributor.context, + workspaceId: 'workspace-1', + userId: 'user-1', + contentUpdatedAt: CONTENT_UPDATED_AT, + }) + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('rendered image content'), + contentType: 'text/plain', + contributingFiles: [contributor], + }) + }) + + function renderedRequest(operation: 'write' | 'compress' | 'content') { + return createMockRequest( + 'POST', + { + operation, + workspaceId: 'workspace-1', + ...(operation === 'write' + ? { + fileName: 'copy.txt', + fileInput: { + id: 'document', + name: 'document.txt', + key: 'workspace/workspace-1/document.txt', + url: '/api/files/serve/document', + context: 'workspace', + size: 1, + type: 'text/plain', + }, + } + : { fileId: 'document' }), + }, + PRIVATE_REQUEST_HEADER + ) + } + + it.each(['write', 'compress', 'content'] as const)( + '%s keeps transformed secret-bearing assets unknown even when the source is exact-empty', + async (operation) => { + mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( + async (_workspaceId: string, identity: { fileId: string }) => ({ + status: 'exact', + entries: identity.fileId === contributor.fileId ? secretEntries : [], + }) + ) + + mockDownloadServableFileFromStorage.mockResolvedValueOnce({ + buffer: Buffer.from(''), + contentType: 'text/html', + contributingFiles: [contributor], + }) + const response = await POST(renderedRequest(operation)) + + expect(response.status, await response.clone().text()).toBe(200) + if (operation === 'content') { + await expect(response.json()).resolves.toMatchObject({ + __resolvedSecretTraceProvenance: { + complete: false, + entries: [], + }, + }) + } else { + expect(mockUploadWorkspaceFile.mock.calls[0]?.[5]).toMatchObject({ + secretProvenance: { status: 'unknown' }, + }) + } + expect(mockDownloadServableFileFromStorage).toHaveBeenCalledWith( + expect.anything(), + 'request-1', + expect.anything(), + expect.objectContaining({ + filePrincipal: expect.objectContaining({ subjectUserId: 'user-1' }), + signal: expect.any(AbortSignal), + }) + ) + expect(mockGetBoundWorkspaceFileSecretProvenance).toHaveBeenCalledWith( + 'workspace-1', + contributor + ) + } + ) + + it.each(['write', 'compress', 'content'] as const)( + '%s preserves unknown rendered-asset provenance', + async (operation) => { + mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( + async (_workspaceId: string, identity: { fileId: string }) => + identity.fileId === contributor.fileId + ? { status: 'unknown' } + : { status: 'exact', entries: [] } + ) + + const response = await POST(renderedRequest(operation)) + + expect(response.status, await response.clone().text()).toBe(200) + if (operation === 'content') { + await expect(response.json()).resolves.toMatchObject({ + __resolvedSecretTraceProvenance: { complete: false, entries: [] }, + }) + } else { + expect(mockUploadWorkspaceFile.mock.calls[0]?.[5]).toMatchObject({ + secretProvenance: { status: 'unknown' }, + }) + } + } + ) + + it.each(['write', 'compress', 'content'] as const)( + '%s does not replace an older rendered revision with a safe revision of the same file', + async (operation) => { + const oldRevision = new Date(CONTENT_UPDATED_AT.getTime() - 1_000) + mockDownloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('both old and new image bytes'), + contentType: 'text/plain', + contributingFiles: [{ ...contributor, contentUpdatedAt: oldRevision }, contributor], + }) + mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( + async (_workspaceId: string, identity: { contentUpdatedAt?: Date }) => + identity.contentUpdatedAt?.getTime() === oldRevision.getTime() + ? { status: 'unknown' } + : { status: 'exact', entries: [] } + ) + + const response = await POST(renderedRequest(operation)) + + expect(response.status, await response.clone().text()).toBe(200) + expect(mockGetBoundWorkspaceFileSecretProvenance).toHaveBeenCalledWith('workspace-1', { + ...contributor, + contentUpdatedAt: oldRevision, + }) + if (operation === 'content') { + await expect(response.json()).resolves.toMatchObject({ + __resolvedSecretTraceProvenance: { complete: false, entries: [] }, + }) + } else { + expect(mockUploadWorkspaceFile.mock.calls[0]?.[5]).toMatchObject({ + secretProvenance: { status: 'unknown' }, + }) + } + } + ) + + it.each(['write', 'compress'] as const)( + '%s retains the secret owner guard for rendered contributors', + async (operation) => { + mockGetFileMetadataByKey.mockResolvedValue({ + id: contributor.fileId, + key: contributor.key, + context: contributor.context, + workspaceId: 'workspace-1', + userId: 'other-user', + contentUpdatedAt: CONTENT_UPDATED_AT, + }) + mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( + async (_workspaceId: string, identity: { fileId: string }) => ({ + status: 'exact', + entries: identity.fileId === contributor.fileId ? secretEntries : [], + }) + ) + + const response = await POST(renderedRequest(operation)) + + expect(response.status, await response.clone().text()).toBe(200) + expect(mockUploadWorkspaceFile.mock.calls[0]?.[5]).toMatchObject({ + secretProvenance: { status: 'unknown' }, + }) + } + ) + + it('refuses a rendered contributor whose canonical scope differs', async () => { + mockGetFileMetadataByKey.mockResolvedValue({ + id: contributor.fileId, + key: contributor.key, + context: contributor.context, + workspaceId: 'other-workspace', + userId: 'user-1', + contentUpdatedAt: CONTENT_UPDATED_AT, + }) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ status: 'exact', entries: [] }) + + const response = await POST(renderedRequest('write')) + + expect(response.status).toBe(404) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + }) + it('pins resolved file-input provenance to the captured content revision', async () => { mockResolveWorkspaceFileReference.mockResolvedValue(workspaceFile('file-1')) mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ @@ -1870,7 +2098,14 @@ describe('file manage operations', () => { ) mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + entries: [ + { + name: 'TOKEN', + encryptedValue: 'encrypted-token', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ], }) const response = await POST( @@ -1885,7 +2120,7 @@ describe('file manage operations', () => { expect(body.__resolvedSecretTraceProvenance).toEqual({ version: 1, complete: true, - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + entries: [{ encryptedValue: 'encrypted-token' }], }) }) diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts index 9907e9103c0..5e84bdcac74 100644 --- a/apps/sim/lib/internal/file/operations.ts +++ b/apps/sim/lib/internal/file/operations.ts @@ -19,6 +19,7 @@ import { inspectPrivateSecretProvenanceRequest, isPrivateSecretProvenanceBundleV1, } from '@/lib/execution/model-input-provenance' +import { resolveStoredFileProvenanceSource } from '@/lib/execution/payloads/file-secret-provenance' import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server' import { PRIVATE_TOOL_METADATA_RESPONSE_HEADER, @@ -44,6 +45,8 @@ import type { WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { + getBoundWorkspaceFileSecretProvenance, + mayReadUnrecordedWorkspaceFile, mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenanceIdentity, @@ -198,11 +201,10 @@ const fileInputToUserFile = (fileInput: unknown) => { ? record.fileId.trim() : '' - // Objects with ids are resolved through workspace metadata. This fallback is for - // picker/upload values that only carry storage fields. - if (id) return null - const key = typeof record.key === 'string' ? record.key.trim() : '' + /** Execution ids are not workspace file ids; their storage key carries the run scope. */ + if (id && (!key || tryInferContextFromKey(key) !== 'execution')) return null + const path = typeof record.path === 'string' ? record.path.trim() : '' const url = typeof record.url === 'string' ? record.url.trim() : '' const fileUrl = @@ -217,7 +219,7 @@ const fileInputToUserFile = (fileInput: unknown) => { if (key && !context) return null return { - id: key || fileUrl, + id: id || key || fileUrl, name: typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file', url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '', @@ -284,6 +286,12 @@ const extractFileIdsFromInput = (fileInput: unknown): string[] => { if (typeof input === 'string') return normalizeFileIdList(input) if (input && typeof input === 'object') { const record = input as Record + if ( + typeof record.key === 'string' && + tryInferContextFromKey(record.key.trim()) === 'execution' + ) { + return [] + } if (typeof record.id === 'string') return normalizeFileIdList(record.id) if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId) } @@ -424,15 +432,23 @@ function sliceTextLines( interface ExtractedFileText { text: string truncated: boolean + contributingFiles?: readonly WorkspaceFileSecretProvenanceIdentity[] } const extractUserFileTextContent = async ( userFile: UserFile, - requestId: string + context: FileManageOperationContext ): Promise => { - const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_GET_CONTENT_FILE_BYTES, - }) + const { buffer, contributingFiles } = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { + maxBytes: MAX_GET_CONTENT_FILE_BYTES, + filePrincipal: context.principal, + signal: context.signal, + } + ) const extension = getFileExtension(userFile.name) if (extension && isSupportedFileType(extension)) { @@ -442,7 +458,11 @@ const extractUserFileTextContent = async ( /** Scraped or placeholder output is a failure, not the file's content. */ throw new Error(result.metadata.warning ?? 'Parser returned degraded output') } - return { text: result.content ?? '', truncated: result.metadata?.truncated === true } + return { + text: result.content ?? '', + truncated: result.metadata?.truncated === true, + contributingFiles, + } } catch (error) { logger.warn('Falling back to raw text after parser failure', { name: userFile.name, @@ -452,16 +472,19 @@ const extractUserFileTextContent = async ( } if (isLikelyTextBuffer(buffer)) { - return { text: buffer.toString('utf-8'), truncated: false } + return { text: buffer.toString('utf-8'), truncated: false, contributingFiles } } return { text: `[Binary file: ${userFile.name} (${userFile.type || 'application/octet-stream'}, ${buffer.length} bytes). Cannot extract text content.]`, truncated: false, + contributingFiles, } } export interface FileContentProvenanceSource { + /** Rendering may encode these bytes; original secret literals cannot describe the transformed value. */ + opaque?: boolean identity?: WorkspaceFileSecretProvenanceIdentity ownerUserId?: string } @@ -471,10 +494,17 @@ interface FileContentSource extends FileContentProvenanceSource { } async function bindSelectedContentFile( - principal: Principal, - workspaceId: string, + context: FileManageOperationContext, file: UserFile ): Promise { + const { principal, workspaceId } = context + if (file.key && tryInferContextFromKey(file.key) === 'execution') { + const source = await resolveStoredFileProvenanceSource(file, { + ...context, + userId: context.fileAccessUserId, + }) + return { file, ...source } + } if (!file.key || file.context !== 'workspace') return { file } let metadata: Awaited> @@ -503,6 +533,67 @@ async function bindSelectedContentFile( } } +async function bindSelectedContentFiles( + context: FileManageOperationContext, + files: readonly UserFile[] +): Promise { + const sources: FileContentSource[] = [] + for (const file of files) { + context.signal?.throwIfAborted() + sources.push(await bindSelectedContentFile(context, file)) + } + return sources +} + +/** Preserves the renderer's consumed revision while checking each contributor's current scope. */ +async function bindRenderedContentSources( + context: FileManageOperationContext, + identities: readonly WorkspaceFileSecretProvenanceIdentity[] = [] +): Promise { + const sources: FileContentProvenanceSource[] = [] + for (const identity of identities) { + context.signal?.throwIfAborted() + const canonical = await resolveStoredFileProvenanceSource( + { + key: identity.key, + context: identity.context === 'mothership' ? 'workspace' : identity.context, + }, + { ...context, userId: context.fileAccessUserId } + ) + const matches = + canonical && + canonical.identity.fileId === identity.fileId && + canonical.identity.key === identity.key && + canonical.identity.context === identity.context + sources.push({ + identity, + opaque: true, + ...(matches ? { ownerUserId: canonical.ownerUserId } : {}), + }) + } + return sources +} + +/** Execution identities have already passed the same run capability that authorized their bytes. */ +async function readFileSourceSecretProvenance( + principal: Principal, + workspaceId: string, + identity: WorkspaceFileSecretProvenanceIdentity +): Promise { + if (identity.context === 'execution' || identity.context === 'mothership') { + return getBoundWorkspaceFileSecretProvenance(workspaceId, identity) + } + const { provenance } = await readWorkspaceFileSecretProvenance.execute({ + principal, + input: { + fileId: identity.fileId, + assertedWorkspaceId: workspaceId, + expectedContentUpdatedAt: identity.contentUpdatedAt, + }, + }) + return provenance +} + export async function getFileContentProvenance( principal: Principal, workspaceId: string, @@ -527,27 +618,25 @@ export async function getFileContentProvenance( accumulator.markIncomplete('file-source-unidentified') continue } - const { provenance } = await readWorkspaceFileSecretProvenance.execute({ - principal, - input: { - fileId: source.identity.fileId, - assertedWorkspaceId: workspaceId, - expectedContentUpdatedAt: source.identity.contentUpdatedAt, - }, - }) + const provenance = await readFileSourceSecretProvenance(principal, workspaceId, source.identity) signal?.throwIfAborted() - /** - * `unrecorded` is a more specific `unknown`, and this accumulator has not opted into the - * workspace file surface's policy, so it latches exactly as it did before. - */ - if (provenance.status !== 'exact') { + if (provenance.status === 'unrecorded' && mayReadUnrecordedWorkspaceFile(workspaceId)) continue + if (provenance.status !== 'exact' || (source.opaque && provenance.entries.length > 0)) { accumulator.markIncomplete('workspace-file-provenance-unknown') continue } accumulator.record({ version: 1, complete: true, - entries: [...provenance.entries], + entries: provenance.entries.map((entry) => ({ + encryptedValue: entry.encryptedValue, + ...(entry.name && + scope && + entry.sourceUserId === scope.userId && + entry.sourceWorkspaceId === scope.workspaceId + ? { name: entry.name } + : {}), + })), ...(scope ? { scope } : {}), }) } @@ -678,25 +767,27 @@ async function deriveWorkspaceFileSecretProvenance(options: { principal: Principal workspaceId: string targetOwnerUserId: string - sources: readonly FileContentSource[] + sources: readonly FileContentProvenanceSource[] }): Promise { - const provenances: WorkspaceFileSecretProvenance[] = [] + let combined: WorkspaceFileSecretProvenance = { status: 'exact', entries: [] } for (const source of options.sources) { if (!source.identity || !source.ownerUserId) return { status: 'unknown' } - const { provenance } = await readWorkspaceFileSecretProvenance.execute({ - principal: options.principal, - input: { fileId: source.identity.fileId, assertedWorkspaceId: options.workspaceId }, - }) + const provenance = await readFileSourceSecretProvenance( + options.principal, + options.workspaceId, + source.identity + ) if ( provenance.status === 'exact' && provenance.entries.length > 0 && - source.ownerUserId !== options.targetOwnerUserId + (source.opaque || source.ownerUserId !== options.targetOwnerUserId) ) { return { status: 'unknown' } } - provenances.push(provenance) + combined = mergeWorkspaceFileSecretProvenance(combined, provenance) + if (combined.status === 'unknown') return combined } - return mergeWorkspaceFileSecretProvenance(...provenances) + return combined } export function fileContentJsonResponse( @@ -1097,10 +1188,9 @@ export async function executeFileManageOperation( }, ] }) - const selectedSources = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) - ) + const selectedSources = await bindSelectedContentFiles(context, selectedInputFiles) const sources = canonicalSources.concat(selectedSources) + const provenanceSources: FileContentProvenanceSource[] = [...sources] const contents: string[] = [] const lineRanges: FileContentLineRange[] = [] @@ -1119,7 +1209,14 @@ export async function executeFileManageOperation( }) } - const extracted = await extractUserFileTextContent(source.file, requestId) + const extracted = await extractUserFileTextContent(source.file, context) + if (includePrivateContentProvenance) { + const renderedSources = await bindRenderedContentSources( + context, + extracted.contributingFiles + ) + for (const renderedSource of renderedSources) provenanceSources.push(renderedSource) + } const { text: content, range } = sliceTextLines( extracted.text, body.offset, @@ -1144,7 +1241,7 @@ export async function executeFileManageOperation( logger.info('File content extracted', { count: contents.length }) const provenance = includePrivateContentProvenance - ? await getFileContentProvenance(principal, workspaceId, sources, signal) + ? await getFileContentProvenance(principal, workspaceId, provenanceSources, signal) : undefined return contentResponse( @@ -1186,8 +1283,8 @@ export async function executeFileManageOperation( * "safe" state — and a file the platform had locked as secret-derived * would be readable again under its new id. * - * A source with no workspace row resolves to `unknown` rather than empty, - * because nothing durable records what went into it. + * Workspace and execution files carry their canonical sidecars across the copy. + * An unidentified source cannot establish exact provenance. */ let inputProvenance: WorkspaceFileSecretProvenance | undefined if (fileInput !== undefined && fileInput !== null) { @@ -1220,12 +1317,7 @@ export async function executeFileManageOperation( const denied = await assertOperationFileAccess(sourceFile, context) if (denied) return denied - inputProvenance = await deriveWorkspaceFileSecretProvenance({ - principal, - workspaceId, - targetOwnerUserId: userId, - sources: [await bindSelectedContentFile(principal, workspaceId, sourceFile)], - }) + const source = await bindSelectedContentFile(context, sourceFile) const downloaded = await downloadServableFileFromStorage(sourceFile, requestId, logger, { maxBytes: MAX_WRITE_FILE_INPUT_BYTES, @@ -1235,6 +1327,15 @@ export async function executeFileManageOperation( // already-published artifact and throws when there is none. filePrincipal: principal, }) + inputProvenance = await deriveWorkspaceFileSecretProvenance({ + principal, + workspaceId, + targetOwnerUserId: userId, + sources: [ + source, + ...(await bindRenderedContentSources(context, downloaded.contributingFiles)), + ], + }) sourceEncoding = 'base64' sourceContent = downloaded.buffer.toString('base64') sourceName = fileName?.trim() || sourceFile.name @@ -1735,16 +1836,19 @@ export async function executeFileManageOperation( return [ { file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, + identity: { + fileId: file.id, + key: file.key, + context: 'workspace', + contentUpdatedAt: file.contentUpdatedAt ?? undefined, + }, ownerUserId: file.uploadedBy, }, ] }) - const selectedArchiveSources = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) - ) + const selectedArchiveSources = await bindSelectedContentFiles(context, selectedInputFiles) const archiveSources = canonicalArchiveSources.concat(selectedArchiveSources) - const archiveProvenance = await deriveWorkspaceFileSecretProvenance({ + let archiveProvenance = await deriveWorkspaceFileSecretProvenance({ principal, workspaceId, targetOwnerUserId: userId, @@ -1775,9 +1879,26 @@ export async function executeFileManageOperation( // the archive must carry the servable bytes instead of the raw source text. // A still-compiling artifact throws, and the handler's catch turns that into // the shared 409 via `docNotReadyResponse`. - const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_COMPRESS_FILE_BYTES, - }) + const { buffer, contributingFiles } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { + maxBytes: MAX_COMPRESS_FILE_BYTES, + filePrincipal: principal, + signal, + } + ) + const renderedSources = await bindRenderedContentSources(context, contributingFiles) + archiveProvenance = mergeWorkspaceFileSecretProvenance( + archiveProvenance, + await deriveWorkspaceFileSecretProvenance({ + principal, + workspaceId, + targetOwnerUserId: userId, + sources: renderedSources, + }) + ) totalBytes += buffer.length if (totalBytes > MAX_COMPRESS_TOTAL_BYTES) { return Response.json( @@ -1905,14 +2026,17 @@ export async function executeFileManageOperation( return [ { file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, + identity: { + fileId: file.id, + key: file.key, + context: 'workspace', + contentUpdatedAt: file.contentUpdatedAt ?? undefined, + }, ownerUserId: file.uploadedBy, }, ] }) - const selectedArchiveSource = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) - ) + const selectedArchiveSource = await bindSelectedContentFiles(context, selectedInputFiles) const archiveSource = canonicalArchiveSource.concat(selectedArchiveSource)[0] if (!archiveSource?.identity) { const denied = await assertOperationFileAccess(archive, context) diff --git a/apps/sim/lib/internal/file/parser.test.ts b/apps/sim/lib/internal/file/parser.test.ts index 9df0f56b89f..c3018aeef8c 100644 --- a/apps/sim/lib/internal/file/parser.test.ts +++ b/apps/sim/lib/internal/file/parser.test.ts @@ -37,6 +37,11 @@ const { mockUploadExecutionFile, mockUploadWorkspaceFile, mockReadWorkspaceFileNameByKey, + mockResolveProvenanceSource, + mockGetBoundProvenance, + mockGetFileContentProvenance, + storageConfig, + mockGetBlobContainerClient, } = vi.hoisted(() => { // eslint-disable-next-line @typescript-eslint/no-require-imports const actualPath = require('path') as typeof import('path') @@ -80,9 +85,40 @@ const { }) ), mockReadWorkspaceFileNameByKey: vi.fn(), + mockResolveProvenanceSource: vi.fn(), + mockGetBoundProvenance: vi.fn(), + mockGetFileContentProvenance: vi.fn(), + storageConfig: { + provider: 's3', + bucket: 'sim-execution-files', + containerName: 'execution-files', + }, + mockGetBlobContainerClient: vi.fn(), } }) +vi.mock('@/lib/execution/payloads/file-secret-provenance', () => ({ + resolveStoredFileProvenanceSource: mockResolveProvenanceSource, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + getBoundWorkspaceFileSecretProvenance: mockGetBoundProvenance, +})) + +vi.mock('@/lib/internal/file/operations', () => ({ + getFileContentProvenance: mockGetFileContentProvenance, + fileContentJsonResponse: ( + body: Record, + includePrivate: boolean, + init?: ResponseInit, + provenance?: unknown + ) => + Response.json( + includePrivate ? { ...body, __resolvedSecretTraceProvenance: provenance } : body, + init + ), +})) + vi.mock('@/lib/execution/payloads/materialization.server', () => ({ assertUserFileContentAccess: async (file: { key: string }) => { if (!(await mockVerifyFileAccess(file.key))) throw new Error('File not found') @@ -95,6 +131,24 @@ vi.mock('@/lib/uploads', () => ({ StorageService: storageServiceMock, })) +vi.mock('@/lib/uploads/config', () => ({ + getStorageConfig: () => storageConfig, + S3_CONFIG: {}, + get USE_S3_STORAGE() { + return storageConfig.provider === 's3' + }, + get USE_BLOB_STORAGE() { + return storageConfig.provider === 'blob' + }, + get USE_GCS_STORAGE() { + return storageConfig.provider === 'gcs' + }, +})) + +vi.mock('@/lib/uploads/providers/blob/client', () => ({ + getBlobServiceClient: async () => ({ getContainerClient: mockGetBlobContainerClient }), +})) + vi.mock('@/lib/file-parsers', () => ({ isSupportedFileType: mockIsSupportedFileType, parseBuffer: mockParseBuffer, @@ -186,6 +240,7 @@ async function POST(request: NextRequest): Promise { executionId: parsed.data.executionId || 'execution-id', attributedUserId: 'test-user-id', fileAccessUserId: 'test-user-id', + headers: request.headers, signal: request.signal, }) } @@ -233,6 +288,8 @@ describe('file parser operation', () => { beforeEach(() => { vi.clearAllMocks() + storageConfig.provider = 's3' + mockGetBlobContainerClient.mockReset() setupFileApiMocks({ authenticated: true, }) @@ -254,6 +311,9 @@ describe('file parser operation', () => { }) mockUploadWorkspaceFile.mockClear() mockReadWorkspaceFileNameByKey.mockResolvedValue({ name: null }) + mockResolveProvenanceSource.mockResolvedValue(undefined) + mockGetBoundProvenance.mockResolvedValue({ status: 'exact', entries: [] }) + mockGetFileContentProvenance.mockResolvedValue({ version: 1, complete: true, entries: [] }) mockParseBuffer.mockResolvedValue({ content: 'parsed buffer content', metadata: { pageCount: 1 }, @@ -278,6 +338,321 @@ describe('file parser operation', () => { expect(data).toHaveProperty('error', 'No file path provided') }) + it('exports negotiated canonical execution-file lineage without changing public content', async () => { + const source = { + identity: { + fileId: 'canonical-file', + key: 'execution/workspace-id/workflow-id/execution-id/report.txt', + context: 'execution', + contentUpdatedAt: new Date('2026-09-10T00:00:00Z'), + }, + ownerUserId: 'test-user-id', + } + mockResolveProvenanceSource.mockResolvedValue(source) + const lineage = { + version: 1, + complete: true, + scope: { userId: 'test-user-id', workspaceId: 'workspace-id' }, + entries: [{ name: 'SECRET', encryptedValue: 'encrypted-value' }], + } + mockGetFileContentProvenance.mockResolvedValue(lineage) + + const response = await POST( + createMockRequest( + 'POST', + { filePath: source.identity.key }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + const body = await response.json() + + expect(body.output.content).toBe('parsed buffer content') + expect(body.__resolvedSecretTraceProvenance).toEqual(lineage) + expect(body).not.toHaveProperty('provenanceSource') + expect(body.output).not.toHaveProperty('provenanceSource') + expect(mockGetFileContentProvenance).toHaveBeenCalledWith( + expect.any(Object), + 'workspace-id', + [source], + expect.any(AbortSignal) + ) + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + }) + + it('keeps private canonical provenance out of unnegotiated parser responses', async () => { + mockResolveProvenanceSource.mockResolvedValue({ + identity: { fileId: 'canonical-file', key: 'workspace/report.txt', context: 'workspace' }, + ownerUserId: 'test-user-id', + }) + const response = await POST(createMockRequest('POST', { filePath: 'workspace/report.txt' })) + const body = await response.json() + + expect(body.output.content).toBe('parsed buffer content') + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(body).not.toHaveProperty('provenanceSource') + expect(mockGetFileContentProvenance).not.toHaveBeenCalled() + }) + + it.each([ + { ownerUserId: 'test-user-id', expectedStatus: 'exact' }, + { ownerUserId: 'other-user', expectedStatus: 'unknown' }, + ])('preserves safe copy provenance for $ownerUserId', async ({ ownerUserId, expectedStatus }) => { + const source = { + identity: { fileId: 'canonical-file', key: 'workspace/report.txt', context: 'workspace' }, + ownerUserId, + } + const entries = [{ name: 'SECRET', encryptedValue: 'encrypted-value' }] + mockResolveProvenanceSource.mockResolvedValue(source) + mockGetBoundProvenance.mockResolvedValue({ status: 'exact', entries }) + + await POST(createMockRequest('POST', { filePath: source.identity.key })) + + expect(mockGetBoundProvenance).toHaveBeenCalledWith('workspace-id', source.identity) + expect(mockUploadExecutionFile).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Buffer), + 'report.txt', + 'text/plain', + 'test-user-id', + expectedStatus === 'exact' ? { status: 'exact', entries } : { status: 'unknown' } + ) + }) + + it('keeps tracked unknown sources in the private response instead of treating them as legacy', async () => { + const source = { + identity: { fileId: 'canonical-file', key: 'workspace/report.txt', context: 'workspace' }, + ownerUserId: 'test-user-id', + } + mockResolveProvenanceSource.mockResolvedValue(source) + mockGetBoundProvenance.mockResolvedValue({ status: 'unknown' }) + mockGetFileContentProvenance.mockResolvedValue({ version: 1, complete: false, entries: [] }) + + const response = await POST( + createMockRequest( + 'POST', + { filePath: source.identity.key }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + + expect((await response.json()).__resolvedSecretTraceProvenance.complete).toBe(false) + expect(mockGetFileContentProvenance).toHaveBeenCalledWith( + expect.any(Object), + 'workspace-id', + [source], + expect.any(AbortSignal) + ) + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'unknown' }) + }) + + it('preserves missing historical metadata as absence on copied files', async () => { + const response = await POST( + createMockRequest( + 'POST', + { filePath: 'workspace/legacy.txt' }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + + expect((await response.json()).success).toBe(true) + expect(mockGetBoundProvenance).not.toHaveBeenCalled() + expect(mockUploadExecutionFile.mock.calls[0][5]).toEqual({ status: 'unrecorded' }) + expect(mockGetFileContentProvenance).toHaveBeenCalledWith( + expect.any(Object), + 'workspace-id', + [], + expect.any(AbortSignal) + ) + }) + + it('does not return content when canonical provenance resolution rejects the file scope', async () => { + mockResolveProvenanceSource.mockRejectedValue(new Error('File not found')) + const response = await POST(createMockRequest('POST', { filePath: 'workspace/other.txt' })) + + expect((await response.json()).success).toBe(false) + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + }) + + it('reads owned presigned URLs through canonical authorized storage', async () => { + const key = 'execution/workspace-id/workflow-id/execution-id/report.txt' + const source = { + identity: { fileId: 'canonical-file', key, context: 'execution' }, + ownerUserId: 'test-user-id', + } + mockResolveProvenanceSource.mockResolvedValue(source) + const response = await POST( + createMockRequest( + 'POST', + { filePath: `https://sim-execution-files.s3.us-east-1.amazonaws.com/${key}?signature=old` }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + + expect((await response.json()).success).toBe(true) + expect(mockResolveProvenanceSource).toHaveBeenCalledWith( + { key, context: 'execution' }, + expect.objectContaining({ workspaceId: 'workspace-id', executionId: 'execution-id' }) + ) + expect(storageServiceMockFns.mockDownloadFile).toHaveBeenCalledWith({ + key, + context: 'execution', + maxBytes: 100 * 1024 * 1024, + }) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it.each([ + 'https://exampleaccount.blob.core.windows.net/execution-files', + 'https://exampleaccount.blob.core.usgovcloudapi.net/execution-files', + 'https://storage.example.test/account/execution-files', + ])( + 'recognizes the configured Azure container endpoint without a separate account name: %s', + async (containerUrl) => { + storageConfig.provider = 'blob' + mockGetBlobContainerClient.mockReturnValue({ url: containerUrl }) + const key = 'execution/workspace-id/workflow-id/execution-id/report.txt' + const source = { + identity: { fileId: 'canonical-file', key, context: 'execution' }, + ownerUserId: 'test-user-id', + } + mockResolveProvenanceSource.mockResolvedValue(source) + mockGetBoundProvenance.mockResolvedValue({ status: 'unknown' }) + + const response = await POST( + createMockRequest('POST', { + filePath: `${containerUrl}/${key}?sig=placeholder`, + }) + ) + + expect((await response.json()).success).toBe(true) + expect(mockGetBlobContainerClient).toHaveBeenCalledWith('execution-files') + expect(mockResolveProvenanceSource).toHaveBeenCalledWith( + { key, context: 'execution' }, + expect.objectContaining({ workspaceId: 'workspace-id' }) + ) + expect(mockGetBoundProvenance).toHaveBeenCalledWith('workspace-id', source.identity) + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + expect(storageServiceMockFns.mockDownloadFile).toHaveBeenCalledWith({ + key, + context: 'execution', + maxBytes: 100 * 1024 * 1024, + }) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + } + ) + + it.each([ + 'https://exampleaccount.blob.core.windows.net.attacker.test/execution-files', + 'https://exampleaccount.blob.core.windows.net/execution-files-other', + ])( + 'does not attribute another Azure origin or container to owned storage: %s', + async (containerUrl) => { + storageConfig.provider = 'blob' + mockGetBlobContainerClient.mockReturnValue({ + url: 'https://exampleaccount.blob.core.windows.net/execution-files', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('external content', { headers: { 'content-type': 'text/plain' } }) + ) + await POST(createMockRequest('POST', { filePath: `${containerUrl}/report.txt` })) + + expect(mockResolveProvenanceSource).not.toHaveBeenCalled() + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + } + ) + + it('does not attribute an external hostname prefix to canonical storage provenance', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('external content', { headers: { 'content-type': 'text/plain' } }) + ) + await POST( + createMockRequest('POST', { + filePath: + 'https://sim-execution-files.s3.us-east-1.amazonaws.com.attacker.test/execution/workspace-id/workflow-id/execution-id/report.txt', + }) + ) + + expect(mockResolveProvenanceSource).not.toHaveBeenCalled() + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + }) + + it('retains only returned contributors when the multi-file output budget stops parsing', async () => { + const first = { + identity: { fileId: 'first', key: 'workspace/first.txt', context: 'workspace' }, + ownerUserId: 'test-user-id', + } + const second = { + identity: { fileId: 'second', key: 'workspace/second.txt', context: 'workspace' }, + ownerUserId: 'test-user-id', + } + mockResolveProvenanceSource.mockResolvedValueOnce(first).mockResolvedValueOnce(second) + mockParseBuffer + .mockResolvedValueOnce({ content: 'a'.repeat(3 * 1024 * 1024) }) + .mockResolvedValueOnce({ content: 'b'.repeat(3 * 1024 * 1024) }) + const response = await POST( + createMockRequest( + 'POST', + { filePath: ['workspace/first.txt', 'workspace/second.txt', 'workspace/third.txt'] }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + const body = await response.json() + + expect(body.success).toBe(true) + expect(body.results).toHaveLength(1) + expect(body.error).toContain('too large') + expect(mockResolveProvenanceSource).toHaveBeenCalledTimes(2) + expect(mockGetFileContentProvenance).toHaveBeenCalledWith( + expect.any(Object), + 'workspace-id', + [first], + expect.any(AbortSignal) + ) + }) + + it.each([{ filePath: 'workspace/failed.txt' }, { filePath: ['workspace/failed.txt'] }])( + 'keeps failed parser provenance out of the public payload for %j', + async ({ filePath }) => { + mockResolveProvenanceSource.mockResolvedValue({ + identity: { fileId: 'private-source', key: 'workspace/failed.txt', context: 'workspace' }, + ownerUserId: 'private-owner', + }) + mockGetBoundProvenance.mockResolvedValue({ + status: 'exact', + entries: [{ name: 'SECRET', encryptedValue: 'private-ciphertext' }], + }) + mockParseBuffer.mockResolvedValue({ + content: 'discarded parser output', + metadata: { degraded: true, warning: 'Unable to parse format' }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { filePath }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1' } + ) + ) + const body = await response.json() + const serialized = JSON.stringify(body) + + expect(Array.isArray(filePath) ? body.results[0].success : body.success).toBe(false) + for (const privateValue of [ + 'provenanceSource', + 'private-source', + 'private-owner', + 'private-ciphertext', + 'discarded parser output', + ]) { + expect(serialized).not.toContain(privateValue) + } + for (const call of mockGetFileContentProvenance.mock.calls) { + expect(call[2]).toEqual([]) + } + } + ) + it('should accept and process a local file', async () => { setupFileApiMocks({ cloudEnabled: false, @@ -458,7 +833,8 @@ describe('file parser operation', () => { parsedBuffer, 'report.pdf', 'application/pdf', - 'test-user-id' + 'test-user-id', + { status: 'unrecorded' } ) }) diff --git a/apps/sim/lib/internal/file/parser.ts b/apps/sim/lib/internal/file/parser.ts index 3318736429b..19f24e02ef7 100644 --- a/apps/sim/lib/internal/file/parser.ts +++ b/apps/sim/lib/internal/file/parser.ts @@ -7,6 +7,7 @@ import type { Principal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' +import { omit } from '@sim/utils/object' import binaryExtensionsList from 'binary-extensions' import type { ContractBody } from '@/lib/api/contracts' import type { fileParseContract } from '@/lib/api/contracts/storage-transfer' @@ -16,18 +17,32 @@ import { isPayloadSizeLimitError, readNodeStreamToBufferWithLimit, } from '@/lib/core/utils/stream-limits' +import { resolveStoredFileProvenanceSource } from '@/lib/execution/payloads/file-secret-provenance' import { assertUserFileContentAccess, type ExecutionMaterializationContext, } from '@/lib/execution/payloads/materialization.server' +import { + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' import { isFileParserError } from '@/lib/file-parsers/errors' +import { + type FileContentProvenanceSource, + fileContentJsonResponse, + getFileContentProvenance, +} from '@/lib/internal/file/operations' import { isUsingCloudStorage, StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { ExternalUrlValidationError, fetchExternalUrlToWorkspace, } from '@/lib/uploads/contexts/workspace' +import { + getBoundWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' import { @@ -74,6 +89,7 @@ export interface FileParserOperationContext { fileKeys?: string[] allowLargeValueWorkflowScope?: boolean requestId?: string + headers?: Headers signal?: AbortSignal } @@ -91,6 +107,8 @@ interface ParseResult { originalName?: string // Original filename from database (for workspace files) viewerUrl?: string | null // Viewer URL for the file if available userFile?: UserFile // UserFile object for the raw file + /** Canonical lineage used only when presenting the private tool response. */ + provenanceSource?: FileContentProvenanceSource metadata?: { fileType: string size: number @@ -103,6 +121,32 @@ function getContentBytes(content: unknown): number { return typeof content === 'string' ? Buffer.byteLength(content, 'utf8') : 0 } +/** Keeps stored source lineage on byte-for-byte copies, including legacy absence. */ +async function resolveParserFileProvenance( + file: Pick, + access: FileReadAccessContext, + targetOwnerUserId: string +): Promise<{ + source?: FileContentProvenanceSource + copyProvenance: WorkspaceFileSecretProvenance +}> { + const source = await resolveStoredFileProvenanceSource(file, access) + if (!source) return { copyProvenance: { status: 'unrecorded' } } + const provenance = await getBoundWorkspaceFileSecretProvenance( + access.workspaceId, + source.identity + ) + return { + source, + copyProvenance: + provenance.status === 'exact' && + provenance.entries.length > 0 && + source.ownerUserId !== targetOwnerUserId + ? { status: 'unknown' } + : provenance, + } +} + export async function executeFileParserOperation( input: FileParserOperationInput, context: FileParserOperationContext @@ -122,6 +166,24 @@ export async function executeFileParserOperation( return Response.json({ success: false, error: 'Execution access denied' }, { status: 403 }) } const { attributedUserId, workspaceId } = context + const sources: FileContentProvenanceSource[] = [] + const includePrivateProvenance = Boolean( + context.headers && + requestsPrivateToolMetadata(context.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + ) + const contentResponse = async (body: Record, init?: ResponseInit) => + fileContentJsonResponse( + body, + includePrivateProvenance, + init, + includePrivateProvenance + ? await getFileContentProvenance(context.principal, workspaceId, sources, context.signal) + : undefined + ) + const partialResponse = async (results: unknown[]) => { + const response = parsedOutputTooLargeResponse(results) + return contentResponse(await response.json(), { status: response.status }) + } const fileReadAccess: FileReadAccessContext = { principal: context.principal, workspaceId, @@ -168,7 +230,7 @@ export async function executeFileParserOperation( const remainingOutputBytes = MAX_MULTI_FILE_PARSE_OUTPUT_BYTES - totalOutputBytes if (remainingOutputBytes <= 0) { - return parsedOutputTooLargeResponse(results) + return await partialResponse(results) } const result = await parseFileSingle( @@ -191,8 +253,9 @@ export async function executeFileParserOperation( if (result.success) { totalOutputBytes += getContentBytes(result.content) if (totalOutputBytes > MAX_MULTI_FILE_PARSE_OUTPUT_BYTES) { - return parsedOutputTooLargeResponse(results) + return await partialResponse(results) } + if (result.provenanceSource) sources.push(result.provenanceSource) const displayName = result.originalName || extractCleanFilename(result.filePath) || 'unknown' @@ -213,13 +276,13 @@ export async function executeFileParserOperation( } if (result.error?.startsWith('Parsed file output is too large')) { - return parsedOutputTooLargeResponse(results) + return await partialResponse(results) } - results.push(result) + results.push(omit(result, ['provenanceSource'])) } - return Response.json({ + return await contentResponse({ success: true, results, }) @@ -242,8 +305,9 @@ export async function executeFileParserOperation( } if (result.success) { + if (result.provenanceSource) sources.push(result.provenanceSource) const displayName = result.originalName || extractCleanFilename(result.filePath) || 'unknown' - return Response.json({ + return await contentResponse({ success: true, output: { content: result.content, @@ -258,7 +322,7 @@ export async function executeFileParserOperation( }) } - return Response.json(result) + return Response.json(omit(result, ['provenanceSource'])) } catch (error) { logger.error('Error in file parse API:', error) return Response.json( @@ -337,6 +401,7 @@ async function parseFileSingle( fileType, workspaceId, attributedUserId, + fileReadAccess, executionContext, headers, signal, @@ -498,15 +563,15 @@ function validateFilePath(filePath: string): { isValid: boolean; error?: string * so keying a cache by filename returns stale bytes. `fetchExternalUrlToWorkspace` * delegates to `uploadWorkspaceFile`, which suffix-disambiguates collisions on save. * - * Workspace save is skipped when the URL already points at our execution-files - * bucket (re-uploading our own bytes is wasteful and would generate `image (1).png` - * style aliases for files we already own). + * URLs for our execution-files storage resolve through the authorized canonical + * read path, keeping stored provenance bound to the same bytes the parser reads. */ async function handleExternalUrl( url: string, fileType: string, workspaceId: string, userId: string, + fileReadAccess: FileReadAccessContext, executionContext?: ExecutionContext, headers?: Record, signal?: AbortSignal, @@ -516,36 +581,81 @@ async function handleExternalUrl( try { logger.info('Fetching external URL:', url) - const { getStorageConfig, USE_S3_STORAGE, USE_BLOB_STORAGE, USE_GCS_STORAGE } = await import( - '@/lib/uploads/config' - ) + const { getStorageConfig, S3_CONFIG, USE_S3_STORAGE, USE_BLOB_STORAGE, USE_GCS_STORAGE } = + await import('@/lib/uploads/config') const executionConfig = getStorageConfig('execution') - let isExecutionFile = false + let executionFileKey: string | undefined try { const parsedUrl = new URL(url) if (USE_S3_STORAGE && executionConfig.bucket) { - const bucketInHost = parsedUrl.hostname.startsWith(executionConfig.bucket) - const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) - isExecutionFile = bucketInHost || bucketInPath + const endpointHost = S3_CONFIG.endpoint ? new URL(S3_CONFIG.endpoint).host : undefined + const bucketHostPrefix = `${executionConfig.bucket}.` + const storageHost = parsedUrl.host.startsWith(bucketHostPrefix) + ? parsedUrl.host.slice(bucketHostPrefix.length) + : parsedUrl.host + const matchesStorageHost = endpointHost + ? storageHost === endpointHost + : /^s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com$/.test(storageHost) + const bucketInHost = matchesStorageHost && parsedUrl.host.startsWith(bucketHostPrefix) + const bucketInPath = + matchesStorageHost && parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) + if (bucketInHost || bucketInPath) { + executionFileKey = decodeURIComponent( + bucketInHost + ? parsedUrl.pathname.slice(1) + : parsedUrl.pathname.slice(executionConfig.bucket.length + 2) + ) + } } else if (USE_BLOB_STORAGE && executionConfig.containerName) { - isExecutionFile = url.includes(`/${executionConfig.containerName}/`) + const { getBlobServiceClient } = await import('@/lib/uploads/providers/blob/client') + const client = await getBlobServiceClient() + const containerUrl = new URL(client.getContainerClient(executionConfig.containerName).url) + const prefix = `${containerUrl.pathname.replace(/\/$/, '')}/` + if (parsedUrl.origin === containerUrl.origin && parsedUrl.pathname.startsWith(prefix)) { + executionFileKey = decodeURIComponent(parsedUrl.pathname.slice(prefix.length)) + } } else if (USE_GCS_STORAGE && executionConfig.bucket) { - const bucketInHost = parsedUrl.hostname.startsWith(`${executionConfig.bucket}.`) - const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) - isExecutionFile = bucketInHost || bucketInPath + const bucketInHost = + parsedUrl.hostname === `${executionConfig.bucket}.storage.googleapis.com` + const bucketInPath = + parsedUrl.hostname === 'storage.googleapis.com' && + parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) + if (bucketInHost || bucketInPath) { + executionFileKey = decodeURIComponent( + bucketInHost + ? parsedUrl.pathname.slice(1) + : parsedUrl.pathname.slice(executionConfig.bucket.length + 2) + ) + } } } catch (error) { logger.warn('Failed to parse URL for execution file check:', error) - isExecutionFile = false + executionFileKey = undefined + } + + /** Read owned storage through its authorized, canonical bytes and provenance together. */ + if (executionFileKey) { + return handleCloudFile( + executionFileKey, + fileType, + userId, + fileReadAccess, + fileReadAccess.principal, + workspaceId, + executionContext, + signal, + maxDownloadBytes, + maxParsedOutputBytes + ) } const { filename, buffer, mimeType } = await fetchExternalUrlToWorkspace({ url, userId, workspaceId: workspaceId || undefined, - saveToWorkspace: Boolean(workspaceId) && !isExecutionFile, + saveToWorkspace: Boolean(workspaceId), headers, signal, maxDownloadBytes, @@ -558,7 +668,9 @@ async function handleExternalUrl( let userFile: UserFile | undefined if (executionContext) { try { - userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId) + userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId, { + status: 'unrecorded', + }) logger.info(`Stored file in execution storage: ${filename}`, { key: userFile.key }) } catch (uploadError) { logger.warn('Failed to store file in execution storage:', uploadError) @@ -672,6 +784,12 @@ async function handleCloudFile( } } + const sourceProvenance = await resolveParserFileProvenance( + { key: cloudKey, context }, + fileReadAccess, + attributedUserId + ) + let originalFilename: string | undefined // Not filtered to `context = 'workspace'`: a chat attachment carries the same key // prefix and has an `originalName` worth recovering too, and without it the parse @@ -745,7 +863,8 @@ async function handleCloudFile( fileBuffer, filename, mimeType, - attributedUserId + attributedUserId, + sourceProvenance.copyProvenance ) logger.info(`Copied file to execution storage: ${filename}`, { key: userFile.key }) } catch (uploadError) { @@ -803,6 +922,9 @@ async function handleCloudFile( if (userFile) { parseResult.userFile = userFile } + if (parseResult.success && sourceProvenance.source) { + parseResult.provenanceSource = sourceProvenance.source + } signal?.throwIfAborted() @@ -873,6 +995,12 @@ async function handleLocalFile( } } + const sourceProvenance = await resolveParserFileProvenance( + { key: storageKey, context }, + fileReadAccess, + attributedUserId + ) + const fullPath = path.join(UPLOAD_DIR_SERVER, storageKey) logger.info('Processing local file:', fullPath) @@ -916,7 +1044,8 @@ async function handleLocalFile( fileBuffer, filename, mimeType, - attributedUserId + attributedUserId, + sourceProvenance.copyProvenance ) logger.info(`Stored local file in execution storage: ${filename}`, { key: userFile.key }) } catch (uploadError) { @@ -930,6 +1059,7 @@ async function handleLocalFile( content, filePath, userFile, + provenanceSource: sourceProvenance.source, metadata: { fileType: mimeType, size: fileBuffer.length, diff --git a/apps/sim/lib/internal/function/execute.test.ts b/apps/sim/lib/internal/function/execute.test.ts index a5ed37edef3..676a24ecd0b 100644 --- a/apps/sim/lib/internal/function/execute.test.ts +++ b/apps/sim/lib/internal/function/execute.test.ts @@ -18,6 +18,7 @@ vi.mock('@/lib/function-execution/application/execute-function', () => ({ import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization' import { executeFunctionTool } from '@/lib/internal/function/execute' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' describe('executeFunctionTool', () => { beforeEach(() => { @@ -59,6 +60,10 @@ describe('executeFunctionTool', () => { executionId: 'execution-1', userId: 'workspace-owner', executorDelegationOrigin: origin, + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { + userId: 'workspace-owner', + workspaceId: 'workspace-1', + }), } const headers = new Headers() @@ -92,6 +97,7 @@ describe('executeFunctionTool', () => { userId: undefined, }), headers, + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, }), }) }) diff --git a/apps/sim/lib/internal/function/execute.ts b/apps/sim/lib/internal/function/execute.ts index 4d68ba712a4..291dddd4fa1 100644 --- a/apps/sim/lib/internal/function/execute.ts +++ b/apps/sim/lib/internal/function/execute.ts @@ -71,6 +71,9 @@ export async function executeFunctionTool(input: ExecuteFunctionToolInput): Prom workspaceId: context.workspaceId, body: trustedBody, headers, + ...(context.resolvedSecretTraceRegistry + ? { resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry } + : {}), ...(signal ? { signal } : {}), ...(sandboxProfile ? { sandboxProfile } : {}), }, diff --git a/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts new file mode 100644 index 00000000000..6cbe10aeed6 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts @@ -0,0 +1,491 @@ +/** Real execution-file storage, ZIP extraction, durable provenance, table import, and KB indexing. */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import type { DelegatedPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { withInsertColumns } from '@sim/db/insert-columns' +import { + document, + documentSecretProvenance, + knowledgeBase, + organization, + outboxEvent, + user, + userTableRowSecretProvenance, + userTableRows, + workspace, + workspaceFileColumns, + workspaceFiles, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { eq, inArray, sql } from 'drizzle-orm' +import JSZip from 'jszip' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixtureStorage = vi.hoisted(() => ({ root: '' })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixtureStorage.root + }, +})) +vi.mock('@/lib/embeddings', async () => ({ + ...(await import('@/lib/embeddings/client')), + assertKnowledgeEmbeddingCapacity: async () => {}, + embedKnowledge: async (texts: string[]) => ({ + embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), + totalTokens: texts.length, + billableTokens: 0, + isBYOK: true, + modelName: 'text-embedding-3-small', + pricingId: 'text-embedding-3-small', + }), +})) + +import { fileManageDecompressBodySchema } from '@/lib/api/contracts/tools/file' +import { processOutboxEventById } from '@/lib/core/outbox/service' +import { encryptSecret } from '@/lib/core/security/encryption' +import { isUserFile } from '@/lib/core/utils/user-file' +import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { addWorkspaceFilesToKnowledgeBase } from '@/lib/knowledge/application/add-workspace-files' +import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' +import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' +import { createSingleDocument } from '@/lib/knowledge/documents/service' +import { loadKnowledgeDocumentSecretRegistry } from '@/lib/knowledge/secret-provenance' +import { createTableFromWorkspaceFile } from '@/lib/table/application/workspace-file-imports' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' +import { + deleteWorkspaceFile, + getWorkspaceFile, +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { + filterModelSafeWorkspaceFileAttachments, + getBoundWorkspaceFileSecretProvenance, + isModelSafeWorkspaceFileKey, + isOpaqueWorkspaceFileEgressSafe, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { deleteFile, downloadFile } from '@/lib/uploads/core/storage-service' +import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' +import type { UserFile } from '@/executor/types' + +const fixtures: ReturnType[] = [] +const trackedEventIds: string[] = [] +const REPORT_TEXT = + 'Orion archive import retains verified source bytes through every durable surface.' +const REPORT_CSV = `name,description\nOrion,${REPORT_TEXT}\n` +const FIXTURE_SECRET = 'fixture-resolved-secret-not-a-live-key' + +async function seed() { + const ids = createKnowledgeAclFixtureIds() + fixtures.push(ids) + await seedKnowledgeAclFixture(ids) + return { ...ids, workflowId: generateId(), executionId: generateId() } +} + +type Fixture = Awaited> + +function sessionPrincipal(ids: Fixture) { + return { kind: 'session', userId: ids.aliceId, sessionId: 'fixture-session' } as const +} + +function tablePrincipal(ids: Fixture): DelegatedPrincipal { + const issuedAt = new Date() + return { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: ids.aliceId, + workspaceId: ids.workspaceId, + delegationId: generateId(), + audience: 'sim:tables', + issuedAt, + expiresAt: new Date(issuedAt.getTime() + 5 * 60_000), + } +} + +async function uploadArchive( + ids: Fixture, + provenance?: WorkspaceFileSecretProvenance, + content = REPORT_CSV +) { + const zip = new JSZip() + zip.file('report.csv', content) + return uploadExecutionFile( + ids, + await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE' }), + 'report.zip', + 'application/zip', + ids.aliceId, + provenance + ) +} + +async function decompress(ids: Fixture, archive: UserFile, executionId = ids.executionId) { + return executeFileManageOperation( + fileManageDecompressBodySchema.parse({ + operation: 'decompress', + workspaceId: ids.workspaceId, + fileInput: archive, + }), + { + principal: createWorkspaceFileDelegatedPrincipal({ + serviceId: 'executor', + subjectUserId: ids.aliceId, + workspaceId: ids.workspaceId, + delegationId: generateId(), + executionId, + }), + workspaceId: ids.workspaceId, + attributedUserId: ids.aliceId, + fileAccessUserId: ids.aliceId, + workflowId: ids.workflowId, + executionId, + headers: new Headers(), + requestId: generateId(), + } + ) +} + +async function extract(ids: Fixture, archive: UserFile) { + const response = await decompress(ids, archive) + const body = await response.json() + expect(response.status, JSON.stringify(body)).toBe(200) + expect(body.success).toBe(true) + const candidates: unknown = body.data?.files + if (!Array.isArray(candidates) || !candidates.every(isUserFile)) { + throw new Error('Archive extraction returned invalid file metadata') + } + expect(candidates).toHaveLength(1) + const child = candidates[0] + const record = await getWorkspaceFile(ids.workspaceId, child.id) + if (!record) throw new Error('Extracted file has no canonical workspace record') + const identity = { + fileId: record.id, + key: record.key, + context: 'workspace' as const, + contentUpdatedAt: record.contentUpdatedAt ?? undefined, + } + return { child, record, identity, publicMetadata: JSON.stringify({ archive, body }) } +} + +async function assertBlockedConsumers(ids: Fixture, source: Awaited>) { + expect(await isOpaqueWorkspaceFileEgressSafe(ids.workspaceId, source.identity)).toBe(false) + const imported = await addWorkspaceFilesToKnowledgeBase.execute({ + principal: sessionPrincipal(ids), + input: { knowledgeBaseId: ids.knowledgeBaseId, fileReferences: [source.child.id] }, + }) + expect(imported).toMatchObject({ added: [], failed: [source.child.id] }) + await expect( + createTableFromWorkspaceFile.execute({ + principal: tablePrincipal(ids), + input: { workspaceId: ids.workspaceId, fileReference: source.child.id }, + }) + ).rejects.toThrow('cannot be verified as free of resolved secrets') +} + +beforeAll(() => { + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-execution-archive-provenance-')) +}) +afterAll(async () => { + if (trackedEventIds.length) { + await db.delete(outboxEvent).where(inArray(outboxEvent.id, trackedEventIds)) + } + for (const ids of fixtures) { + await db.delete(knowledgeBase).where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + } + await rm(fixtureStorage.root, { recursive: true, force: true }) + await db.$client.end() +}) + +describe('execution archive durable provenance', () => { + it('carries exact-empty lineage through extraction, table rows, and delayed KB indexing/search', async () => { + const ids = await seed() + const archive = await uploadArchive(ids, { status: 'exact', entries: [] }) + const [storedArchive] = await db + .select({ + secretProvenanceVersion: workspaceFiles.secretProvenanceVersion, + context: workspaceFiles.context, + }) + .from(workspaceFiles) + .where(eq(workspaceFiles.key, archive.key)) + expect(storedArchive.secretProvenanceVersion).toBe(1) + expect(storedArchive.context).toBe('execution') + const source = await extract(ids, archive) + expect(await getBoundWorkspaceFileSecretProvenance(ids.workspaceId, source.identity)).toEqual({ + status: 'exact', + entries: [], + }) + expect(await isOpaqueWorkspaceFileEgressSafe(ids.workspaceId, source.identity)).toBe(true) + expect((await downloadFile({ key: source.child.key, context: 'workspace' })).toString()).toBe( + REPORT_CSV + ) + + const table = await createTableFromWorkspaceFile.execute({ + principal: tablePrincipal(ids), + input: { workspaceId: ids.workspaceId, fileReference: source.child.id }, + }) + expect(table.kind).toBe('inline') + if (table.kind !== 'inline') throw new Error('Small CSV did not use the inline import path') + expect(table.insertedCount).toBe(1) + const rows = await db + .select({ + data: userTableRows.data, + updatedAt: userTableRows.updatedAt, + version: userTableRows.secretProvenanceVersion, + contentUpdatedAt: userTableRowSecretProvenance.contentUpdatedAt, + status: userTableRowSecretProvenance.status, + entries: userTableRowSecretProvenance.entries, + }) + .from(userTableRows) + .leftJoin( + userTableRowSecretProvenance, + eq(userTableRowSecretProvenance.rowId, userTableRows.id) + ) + .where(eq(userTableRows.tableId, table.table.id)) + expect(rows).toHaveLength(1) + const nameColumn = table.table.schema.columns.find((column) => column.name === 'name') + const descriptionColumn = table.table.schema.columns.find( + (column) => column.name === 'description' + ) + if (!nameColumn?.id || !descriptionColumn?.id) { + throw new Error('Imported table lost its canonical source columns') + } + expect(rows[0]).toMatchObject({ + data: { [nameColumn.id]: 'Orion', [descriptionColumn.id]: REPORT_TEXT }, + version: 1, + status: 'exact', + entries: [], + }) + expect(rows[0].contentUpdatedAt).toEqual(rows[0].updatedAt) + + const imported = await addWorkspaceFilesToKnowledgeBase.execute({ + principal: sessionPrincipal(ids), + input: { knowledgeBaseId: ids.knowledgeBaseId, fileReferences: [source.child.id] }, + }) + expect(imported.failed).toEqual([]) + expect(imported.added).toHaveLength(1) + const documentId = imported.added[0].documentId + const [admitted] = await db.select().from(document).where(eq(document.id, documentId)) + expect(admitted.secretProvenanceVersion).toBe(1) + expect(admitted.storageKey).toMatch(/^kb\//) + const events = await db + .select() + .from(outboxEvent) + .where(sql`${outboxEvent.payload}::jsonb ->> 'documentId' = ${documentId}`) + trackedEventIds.push(...events.map((event) => event.id)) + const dispatch = events.find( + (event) => event.eventType === KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT + ) + if (!dispatch) throw new Error('Knowledge import did not atomically admit processing') + await deleteWorkspaceFile(ids.workspaceId, source.child.id) + await deleteFile({ key: source.child.key, context: 'workspace' }) + await deleteFile({ key: archive.key, context: 'execution' }) + await processOutboxEventById(dispatch.id, knowledgeDocumentProcessingOutboxHandlers) + const [indexed] = await db.select().from(document).where(eq(document.id, documentId)) + expect(indexed.processingStatus, indexed.processingError ?? undefined).toBe('completed') + const chunks = await listKnowledgeChunks.execute({ + principal: sessionPrincipal(ids), + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId }, + }) + expect(chunks.chunks.map((chunk) => chunk.content).join('\n')).toContain(REPORT_TEXT) + const search = await searchKnowledge.execute({ + principal: sessionPrincipal(ids), + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + searchMode: 'hybrid', + topK: 10, + }, + }) + expect(search.results.map((entry) => entry.documentId)).toContain(documentId) + }) + + it('keeps an explicitly unknown execution source unavailable to model, KB, and table consumers', async () => { + const ids = await seed() + const archive = await uploadArchive(ids, { status: 'unknown' }) + const source = await extract(ids, archive) + expect(await getBoundWorkspaceFileSecretProvenance(ids.workspaceId, source.identity)).toEqual({ + status: 'unknown', + }) + await assertBlockedConsumers(ids, source) + }) + + it('does not infer safe extracted bytes from a secret-bearing archive or expose private metadata', async () => { + const ids = await seed() + const { encrypted } = await encryptSecret(FIXTURE_SECRET) + const archive = await uploadArchive( + ids, + { + status: 'exact', + entries: [ + { + name: 'FIXTURE_SECRET', + encryptedValue: encrypted, + sourceUserId: ids.aliceId, + sourceWorkspaceId: ids.workspaceId, + }, + ], + }, + `name,description\nOrion,${FIXTURE_SECRET}\n` + ) + const source = await extract(ids, archive) + expect(source.publicMetadata).not.toContain(FIXTURE_SECRET) + expect(source.publicMetadata).not.toContain(encrypted) + expect(source.publicMetadata).not.toContain('encryptedValue') + expect(await getBoundWorkspaceFileSecretProvenance(ids.workspaceId, source.identity)).toEqual({ + status: 'unknown', + }) + await assertBlockedConsumers(ids, source) + }) + + it('preserves compatibility for execution files created before provenance stamping', async () => { + const ids = await seed() + const archive = await uploadArchive(ids) + const [storedArchive] = await db + .select({ + secretProvenanceVersion: workspaceFiles.secretProvenanceVersion, + context: workspaceFiles.context, + }) + .from(workspaceFiles) + .where(eq(workspaceFiles.key, archive.key)) + expect(storedArchive.secretProvenanceVersion).toBeNull() + const source = await extract(ids, archive) + expect(await isOpaqueWorkspaceFileEgressSafe(ids.workspaceId, source.identity)).toBe(true) + const imported = await createTableFromWorkspaceFile.execute({ + principal: tablePrincipal(ids), + input: { workspaceId: ids.workspaceId, fileReference: source.child.id }, + }) + expect(imported.kind).toBe('inline') + }) + + it.each([false, true])( + 'refuses tracked unknown execution attachments with historical metadata (archivedOnly=%s)', + async (archivedOnly) => { + const ids = await seed() + const file = await uploadExecutionFile( + ids, + Buffer.from(REPORT_CSV), + 'report.csv', + 'text/csv', + ids.aliceId, + { status: 'unknown' } + ) + if (archivedOnly) { + await db + .update(workspaceFiles) + .set({ deletedAt: new Date() }) + .where(eq(workspaceFiles.key, file.key)) + } else { + await db.insert(withInsertColumns(workspaceFiles, workspaceFileColumns)).values({ + id: generateId(), + key: file.key, + userId: ids.aliceId, + workspaceId: ids.workspaceId, + context: 'execution', + originalName: 'historical-report.csv', + contentType: file.type, + sizeBytes: file.size, + deletedAt: new Date(), + contentUpdatedAt: new Date(Date.now() + 60_000), + secretProvenanceVersion: null, + }) + } + + expect( + await filterModelSafeWorkspaceFileAttachments([file], { workspaceId: ids.workspaceId }) + ).toEqual([]) + expect(await isModelSafeWorkspaceFileKey(file.key, { workspaceId: ids.workspaceId })).toBe( + false + ) + } + ) + + it.each([ + { status: 'exact', deleted: false }, + { status: 'unknown', deleted: false }, + { status: 'exact', deleted: true }, + { status: 'unknown', deleted: true }, + ] as const)( + 'binds $status execution bytes into KB admission despite URL-only classification (deleted=$deleted)', + async ({ status, deleted }) => { + const ids = await seed() + const file = await uploadExecutionFile( + ids, + Buffer.from(REPORT_CSV), + 'report.csv', + 'text/csv', + ids.aliceId, + status === 'exact' ? { status, entries: [] } : { status } + ) + if (deleted) { + await db + .update(workspaceFiles) + .set({ deletedAt: new Date() }) + .where(eq(workspaceFiles.key, file.key)) + } + const admitted = await createSingleDocument( + { + filename: file.name, + fileUrl: `/api/files/serve/${encodeURIComponent(file.key)}?context=workspace`, + fileSize: file.size, + mimeType: file.type, + }, + ids.knowledgeBaseId, + generateId(), + ids.aliceId, + undefined, + { + filename: { status: 'exact', entries: [] }, + content: { status: 'exact', entries: [] }, + tags: [], + } + ) + const [stored] = await db + .select({ + version: document.secretProvenanceVersion, + status: documentSecretProvenance.status, + }) + .from(document) + .leftJoin(documentSecretProvenance, eq(documentSecretProvenance.documentId, document.id)) + .where(eq(document.id, admitted.id)) + expect(stored).toEqual({ version: 1, status }) + const registry = loadKnowledgeDocumentSecretRegistry(admitted.id, { + userId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + if (status === 'exact') { + await expect(registry).resolves.toMatchObject({ + tracked: true, + provenance: { status: 'exact', entries: [] }, + }) + } else { + await expect(registry).rejects.toThrow( + 'Knowledge document secret provenance is unavailable' + ) + } + } + ) + + it('refuses another execution before extracting any workspace files', async () => { + const ids = await seed() + const archive = await uploadArchive(ids, { status: 'exact', entries: [] }) + const response = await decompress(ids, archive, generateId()) + expect(response.status).toBe(404) + const files = await db + .select({ context: workspaceFiles.context }) + .from(workspaceFiles) + .where(eq(workspaceFiles.workspaceId, ids.workspaceId)) + expect(files).toEqual([{ context: 'execution' }]) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 5b36cbdf3c2..24b77c321fa 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -420,6 +420,142 @@ describe('knowledge document processing source', () => { expect(mockGenerateEmbeddings).not.toHaveBeenCalled() }) + describe('execution source provenance', () => { + const executionKey = 'execution/workspace-1/workflow-1/run-1/source.pdf' + const executionUrl = `/api/files/serve/${encodeURIComponent(executionKey)}?context=workspace` + const executionBinding = { + ...SOURCE_BINDING, + key: executionKey, + context: 'execution', + secretProvenanceVersion: 1, + } + + beforeEach(() => { + dbChainMockFns.limit + .mockReset() + .mockResolvedValueOnce([{ ...PERSISTED_CONTEXT, fileUrl: executionUrl }]) + .mockResolvedValueOnce([{ ...PERSISTED_PROVENANCE_ROW, fileUrl: executionUrl }]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) => + context === 'execution' ? [executionBinding] : [] + ) + }) + + function process() { + return processDocumentAsync( + 'knowledge-base-1', + 'document-1', + { + filename: 'untrusted-queued-name.txt', + fileUrl: 'https://example.com/untrusted-queued-url.txt', + fileSize: 1, + mimeType: 'text/plain', + }, + {}, + BILLING_ATTRIBUTION + ) + } + + it('loads persisted execution lineage before parsing, ignoring the URL context label', async () => { + await process() + + expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith( + [executionKey], + 'execution', + expect.anything(), + { includeDeleted: true } + ) + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith( + expect.anything(), + [executionBinding] + ) + expect(mockProcessDocument).toHaveBeenCalledWith( + executionUrl, + PERSISTED_CONTEXT.filename, + PERSISTED_CONTEXT.mimeType, + 1024, + 200, + 100, + expect.objectContaining({ userId: BILLING_ATTRIBUTION.actorUserId }), + PERSISTED_CONTEXT.workspaceId, + undefined, + undefined + ) + }) + + it.each(['unknown', 'missing'])( + 'refuses tracked execution sources with %s sidecars before parsing', + async (kind) => { + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map(kind === 'unknown' ? [[executionBinding.id, { status: 'unknown' }]] : []) + ) + + await expect(process()).rejects.toThrow( + 'Knowledge document secret provenance is unavailable' + ) + + expect(mockProcessDocument).not.toHaveBeenCalled() + expect(mockGenerateEmbeddings).not.toHaveBeenCalled() + } + ) + + it('refuses a soft-deleted tracked execution source before parsing', async () => { + const deletedBinding = { ...executionBinding, deletedAt: CONTENT_UPDATED_AT } + mockGetFileMetadataByKeys.mockImplementation( + async ( + _keys: string[], + context: string, + _executor: unknown, + options?: { includeDeleted?: boolean } + ) => (context === 'execution' && options?.includeDeleted ? [deletedBinding] : []) + ) + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map([[executionBinding.id, { status: 'unknown' }]]) + ) + + await expect(process()).rejects.toThrow('Knowledge document secret provenance is unavailable') + + expect(mockProcessDocument).not.toHaveBeenCalled() + expect(mockGenerateEmbeddings).not.toHaveBeenCalled() + }) + + it('preserves legacy null-marker behavior for a soft-deleted execution source', async () => { + mockGetFileMetadataByKeys.mockResolvedValue([ + { ...executionBinding, deletedAt: CONTENT_UPDATED_AT, secretProvenanceVersion: null }, + ]) + + await process() + + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).not.toHaveBeenCalled() + expect(mockProcessDocument).toHaveBeenCalled() + }) + + it.each(['missing', 'untracked'])( + 'retains legacy %s execution source behavior', + async (kind) => { + mockGetFileMetadataByKeys.mockResolvedValue( + kind === 'missing' ? [] : [{ ...executionBinding, secretProvenanceVersion: null }] + ) + + await process() + + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).not.toHaveBeenCalled() + expect(mockProcessDocument).toHaveBeenCalled() + } + ) + + it('refuses a source whose execution metadata belongs to another workspace', async () => { + mockGetFileMetadataByKeys.mockResolvedValue([ + { ...executionBinding, workspaceId: 'other-workspace' }, + ]) + + await expect(process()).rejects.toThrow('Document file is not owned by this knowledge base') + + expect(mockProcessDocument).not.toHaveBeenCalled() + expect(mockGenerateEmbeddings).not.toHaveBeenCalled() + }) + }) + it('takes over an existing processing attempt', async () => { dbChainMockFns.limit .mockReset() diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index d3dea4c4ecc..65eb4b14f95 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -184,7 +184,7 @@ const logger = createLogger('DocumentService') /** * Thrown when a knowledge-base document's `fileUrl` references an internal - * knowledge-base storage object not owned by the target knowledge base's workspace. + * knowledge-base or execution object not owned by the target knowledge base's workspace. * Routes map this to a 403. * * Deliberately carries no `details.code`. It belongs to the cross-tenant class @@ -214,12 +214,15 @@ function getKnowledgeBaseStorageKeys(fileUrls: readonly string[]): string[] { ] } -function getWorkspaceSourceStorageKeys(fileUrls: readonly string[]): string[] { +function getSourceStorageKeys( + fileUrls: readonly string[], + context: 'workspace' | 'execution' +): string[] { return [ ...new Set( fileUrls .map((url) => getKnowledgeBaseStorageKey(url)) - .filter((key): key is string => typeof key === 'string' && key.startsWith('workspace/')) + .filter((key): key is string => typeof key === 'string' && key.startsWith(`${context}/`)) ), ] } @@ -238,18 +241,39 @@ async function loadKnowledgeBaseFileBindings( return new Map(bindings.map((binding) => [binding.key, binding])) } -async function loadWorkspaceSourceFileBindings( +/** Execution metadata without a provenance marker predates stamping and remains a legacy source. */ +async function loadSourceFileBindings( fileUrls: readonly string[], + workspaceId: string | null, executor: DbExecutor = db ): Promise> { - const keys = getWorkspaceSourceStorageKeys(fileUrls) - if (keys.length === 0) return new Map() + const workspaceKeys = getSourceStorageKeys(fileUrls, 'workspace') + const executionKeys = getSourceStorageKeys(fileUrls, 'execution') + const workspaceBindings = + workspaceKeys.length > 0 + ? await getFileMetadataByKeys(workspaceKeys, 'workspace', executor) + : [] + const mothershipBindings = + workspaceKeys.length > 0 + ? await getFileMetadataByKeys(workspaceKeys, 'mothership', executor) + : [] + const executionBindings = + executionKeys.length > 0 + ? await getFileMetadataByKeys(executionKeys, 'execution', executor, { includeDeleted: true }) + : [] - const workspaceBindings = await getFileMetadataByKeys(keys, 'workspace', executor) - const mothershipBindings = await getFileMetadataByKeys(keys, 'mothership', executor) + for (const binding of executionBindings) { + if (!workspaceId || binding.workspaceId !== workspaceId) { + throw new KnowledgeBaseFileOwnershipError(binding.key) + } + } return new Map( - [...workspaceBindings, ...mothershipBindings].map((binding) => [binding.key, binding]) + [ + ...workspaceBindings, + ...mothershipBindings, + ...executionBindings.filter((binding) => binding.secretProvenanceVersion !== null), + ].map((binding) => [binding.key, binding]) ) } @@ -281,13 +305,14 @@ async function assertKnowledgeBaseFileUrlsOwnership( return bindingByKey } -async function loadCurrentWorkspaceSourceFileSecretProvenance(options: { +async function loadCurrentSourceFileSecretProvenance(options: { fileUrl: string + workspaceId: string | null }): Promise { const storageKey = getKnowledgeBaseStorageKey(options.fileUrl) - if (!storageKey?.startsWith('workspace/')) return undefined + if (!storageKey) return undefined - const bindingByKey = await loadWorkspaceSourceFileBindings([options.fileUrl]) + const bindingByKey = await loadSourceFileBindings([options.fileUrl], options.workspaceId) const binding = bindingByKey.get(storageKey) if (!binding) return undefined @@ -388,7 +413,7 @@ interface DocumentTagData { type TagDefinition = typeof knowledgeBaseTagDefinitions.$inferSelect type TagDefinitionsByName = Map -type DbExecutor = Pick +type DbExecutor = Pick async function loadTagDefinitions( knowledgeBaseId: string, @@ -1638,8 +1663,9 @@ export async function processDocumentAsync( let embeddingModelName = kbEmbeddingModel let embeddingPricingId = kbEmbeddingModel - const currentSourceFileProvenance = await loadCurrentWorkspaceSourceFileSecretProvenance({ + const currentSourceFileProvenance = await loadCurrentSourceFileSecretProvenance({ fileUrl: persistedDocData.fileUrl, + workspaceId: ctx.workspaceId, }) const documentSecretContext = await loadKnowledgeDocumentSecretRegistry( documentId, @@ -2287,8 +2313,9 @@ export async function createDocumentRecords( requestId, tx ) - const sourceBindingByKey = await loadWorkspaceSourceFileBindings( + const sourceBindingByKey = await loadSourceFileBindings( resolvedDocuments.map((docData) => docData.fileUrl), + admission.workspaceId, tx ) const trackedBindings = [ @@ -2961,8 +2988,9 @@ export async function createSingleDocument( requestId, tx ) - const sourceBindingByKey = await loadWorkspaceSourceFileBindings( + const sourceBindingByKey = await loadSourceFileBindings( [resolvedDocumentData.fileUrl], + admission.workspaceId, tx ) const storageKey = getKnowledgeBaseStorageKey(resolvedDocumentData.fileUrl) diff --git a/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts b/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts index ecbc5e5f543..4c860c16817 100644 --- a/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/workspace-source-provenance.test.ts @@ -3,6 +3,7 @@ */ import { dbChainMockFns, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' const { mockCheckStorageQuotaForBillingContext, @@ -282,6 +283,137 @@ describe('knowledge workspace source provenance', () => { expect(findDocumentProvenanceWrite()).toBeUndefined() }) + describe('execution file sources', () => { + const executionKey = `execution/${WORKSPACE_ID}/workflow-1/run-1/source.pdf` + const executionUrl = `/api/files/serve/${encodeURIComponent(executionKey)}?context=workspace` + const executionBinding = { + ...SOURCE_BINDING, + id: 'execution-source-1', + key: executionKey, + context: 'execution', + } + const documentInput = { + filename: 'source.pdf', + fileUrl: executionUrl, + fileSize: 512, + mimeType: 'application/pdf', + } + + beforeEach(() => { + mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) => + context === 'execution' ? [executionBinding] : [] + ) + }) + + for (const mode of ['single', 'bulk'] as const) { + async function create() { + if (mode === 'single') { + await createSingleDocument(documentInput, KNOWLEDGE_BASE_ID, 'request-1', SOURCE_USER_ID) + } else { + await createDocumentRecords( + [documentInput], + KNOWLEDGE_BASE_ID, + 'request-1', + SOURCE_USER_ID + ) + } + } + + it.each([ + { status: 'exact', entries: [] }, + { + status: 'exact', + entries: [{ name: 'EXPORT_SECRET', encryptedValue: 'encrypted-export-secret' }], + }, + { status: 'unknown' }, + ] satisfies WorkspaceFileSecretProvenance[])( + `binds canonical execution byte provenance during ${mode} admission: %j`, + async (provenance) => { + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map([[executionBinding.id, provenance]]) + ) + + await create() + + expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith( + [executionKey], + 'execution', + expect.anything(), + { includeDeleted: true } + ) + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith( + expect.anything(), + [executionBinding] + ) + expect(findDocumentProvenanceWrite()).toEqual( + expect.objectContaining({ + status: provenance.status, + entries: + provenance.status === 'exact' + ? provenance.entries.map((entry) => + expect.objectContaining({ + ...entry, + sourceUserId: SOURCE_USER_ID, + sourceWorkspaceId: WORKSPACE_ID, + sourceValueHash: expect.any(String), + }) + ) + : [], + }) + ) + } + ) + + it(`preserves soft-deleted execution taint during ${mode} admission`, async () => { + const deletedBinding = { ...executionBinding, deletedAt: CONTENT_UPDATED_AT } + mockGetFileMetadataByKeys.mockImplementation( + async ( + _keys: string[], + context: string, + _executor: unknown, + options?: { includeDeleted?: boolean } + ) => (context === 'execution' && options?.includeDeleted ? [deletedBinding] : []) + ) + mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue( + new Map([[executionBinding.id, { status: 'unknown' }]]) + ) + + await create() + + expect(findDocumentProvenanceWrite()).toMatchObject({ status: 'unknown', entries: [] }) + }) + + it.each(['missing', 'untracked'])( + `preserves legacy %s execution sources during ${mode} admission`, + async (kind) => { + mockGetFileMetadataByKeys.mockResolvedValue( + kind === 'missing' ? [] : [{ ...executionBinding, secretProvenanceVersion: null }] + ) + + await create() + + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith( + expect.anything(), + [] + ) + expect(findDocumentProvenanceWrite()).toBeUndefined() + } + ) + + it(`refuses another workspace's execution source before ${mode} admission`, async () => { + mockGetFileMetadataByKeys.mockResolvedValue([ + { ...executionBinding, workspaceId: 'other-workspace' }, + ]) + + await expect(create()).rejects.toThrow('Document file is not owned by this knowledge base') + + expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).not.toHaveBeenCalled() + expect(findDocumentProvenanceWrite()).toBeUndefined() + expect(mockIncrementStorageUsageForBillingContextInTx).not.toHaveBeenCalled() + }) + } + }) + it('never deletes a referenced workspace source as knowledge-base storage', async () => { await deleteDocumentStorageFiles( [{ id: 'document-1', fileUrl: SOURCE_URL, workspaceId: WORKSPACE_ID }], diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.test.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.test.ts index 4f0c5d007c5..1d108c60d37 100644 --- a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.test.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.test.ts @@ -4,9 +4,10 @@ import { dbChainMockFns, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockUploadToS3, mockGetPresignedUrlWithConfig } = vi.hoisted(() => ({ +const { mockUploadToS3, mockGetPresignedUrlWithConfig, mockDeleteFromS3 } = vi.hoisted(() => ({ mockUploadToS3: vi.fn(), mockGetPresignedUrlWithConfig: vi.fn(), + mockDeleteFromS3: vi.fn(), })) vi.mock('@/lib/uploads/config', () => ({ @@ -19,6 +20,7 @@ vi.mock('@/lib/uploads/config', () => ({ vi.mock('@/lib/uploads/providers/s3/client', () => ({ uploadToS3: mockUploadToS3, getPresignedUrlWithConfig: mockGetPresignedUrlWithConfig, + deleteFromS3: mockDeleteFromS3, })) import { uploadExecutionFile } from '@/lib/uploads/contexts/execution/execution-file-manager' @@ -41,6 +43,7 @@ describe('uploadExecutionFile key allocation', () => { type: contentType, })) mockGetPresignedUrlWithConfig.mockResolvedValue('https://example.com/download') + mockDeleteFromS3.mockResolvedValue(undefined) dbChainMockFns.limit.mockResolvedValue([]) dbChainMockFns.returning.mockResolvedValue([{ id: 'file-1' }]) }) @@ -64,4 +67,104 @@ describe('uploadExecutionFile key allocation', () => { expect(first.key).not.toBe(second.key) expect(dbChainMockFns.insert).toHaveBeenCalledTimes(2) }) + + it('commits tracked provenance with the canonical file before returning its URL', async () => { + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + dbChainMockFns.returning.mockImplementation(async () => { + const values = dbChainMockFns.values.mock.calls.at(-1)?.[0] + return [{ ...values, id: values?.id ?? values?.fileId, contentUpdatedAt }] + }) + const file = await uploadExecutionFile( + context, + Buffer.from('archive'), + 'report.zip', + 'application/zip', + 'user-1', + { status: 'exact', entries: [] } + ) + + expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.values).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ id: file.id, key: file.key, context: 'execution' }) + ) + expect(dbChainMockFns.values).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ fileId: file.id, contentUpdatedAt, status: 'exact', entries: [] }) + ) + expect(dbChainMockFns.set).toHaveBeenCalledWith({ secretProvenanceVersion: 1 }) + expect(dbChainMockFns.set.mock.invocationCallOrder[0]).toBeLessThan( + mockGetPresignedUrlWithConfig.mock.invocationCallOrder[0] + ) + expect(file).not.toHaveProperty('secretProvenance') + }) + + it('removes uploaded bytes when their provenance cannot be committed', async () => { + const failure = new Error('Provenance commit failed') + dbChainMockFns.returning + .mockResolvedValueOnce([ + { + id: 'recorded-file', + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + }, + ]) + .mockRejectedValueOnce(failure) + + await expect( + uploadExecutionFile( + context, + Buffer.from('archive'), + 'report.zip', + 'application/zip', + 'user-1', + { status: 'unknown' } + ) + ).rejects.toThrow('Provenance commit failed') + + expect(mockDeleteFromS3).toHaveBeenCalledWith( + mockUploadToS3.mock.calls[0][1], + expect.any(Object), + undefined + ) + expect(mockGetPresignedUrlWithConfig).not.toHaveBeenCalled() + }) + + it('rejects tracked uploads without an owner before writing bytes', async () => { + await expect( + uploadExecutionFile( + context, + Buffer.from('archive'), + 'report.zip', + 'application/zip', + undefined, + { + status: 'exact', + entries: [], + } + ) + ).rejects.toThrow('requires an owner and workspace') + expect(mockUploadToS3).not.toHaveBeenCalled() + }) + + it('cleans both committed metadata and bytes when its download URL cannot be issued', async () => { + const contentUpdatedAt = new Date('2026-01-01T00:00:00Z') + dbChainMockFns.returning.mockImplementation(async () => { + const values = dbChainMockFns.values.mock.calls.at(-1)?.[0] + return [{ ...values, id: values?.id ?? values?.fileId, contentUpdatedAt }] + }) + mockGetPresignedUrlWithConfig.mockRejectedValueOnce(new Error('Signing failed')) + + await expect( + uploadExecutionFile( + context, + Buffer.from('archive'), + 'report.zip', + 'application/zip', + 'user-1', + { status: 'unknown' } + ) + ).rejects.toThrow('Signing failed') + expect(mockDeleteFromS3).toHaveBeenCalledTimes(1) + expect(dbChainMockFns.set).toHaveBeenCalledWith({ deletedAt: expect.any(Date) }) + }) }) diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts index d4f4bea9cb9..b9bb85213b7 100644 --- a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts @@ -1,5 +1,7 @@ +import { db } from '@sim/db' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import type { ExecutionContext } from '@/lib/uploads/contexts/execution/utils' @@ -7,6 +9,15 @@ import { generateFileId, generateUniqueExecutionFileKey, } from '@/lib/uploads/contexts/execution/utils' +import { + initializeWorkspaceFileSecretProvenanceInTx, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { + deleteFileMetadataByIdentity, + type FileMetadataRecord, + insertImmutableFileMetadata, +} from '@/lib/uploads/server/metadata' import type { UserFile } from '@/executor/types' const logger = createLogger('ExecutionFileStorage') @@ -69,8 +80,12 @@ export async function uploadExecutionFile( fileBuffer: Buffer, fileName: string, contentType: string, - userId?: string + userId?: string, + secretProvenance?: WorkspaceFileSecretProvenance ): Promise { + if (secretProvenance && (!userId || !context.workspaceId)) { + throw new Error('Execution file provenance requires an owner and workspace') + } logger.info(`Uploading execution file: ${fileName} for execution ${context.executionId}`) logger.debug(`File upload context:`, { workspaceId: context.workspaceId, @@ -82,7 +97,7 @@ export async function uploadExecutionFile( }) const storageKey = generateUniqueExecutionFileKey(context, fileName) - const fileId = generateFileId() + const fileId = secretProvenance ? generateId() : generateFileId() logger.info(`Generated storage key: "${storageKey}" for file: ${fileName}`) @@ -97,8 +112,10 @@ export async function uploadExecutionFile( metadata.userId = userId } + const StorageService = await getStorageService() + let uploadedKey: string | undefined + let recordedFile: FileMetadataRecord | undefined try { - const StorageService = await getStorageService() const fileInfo = await StorageService.uploadFile({ file: fileBuffer, fileName: storageKey, @@ -107,7 +124,34 @@ export async function uploadExecutionFile( preserveKey: true, // Don't add timestamp prefix customKey: storageKey, // Use exact execution-scoped key metadata, // Pass metadata for cloud storage and database tracking + ...(secretProvenance ? { persistMetadata: false } : {}), }) + uploadedKey = fileInfo.key + + if (secretProvenance && userId) { + recordedFile = await db.transaction(async (tx) => { + const record = await insertImmutableFileMetadata( + { + id: fileId, + key: fileInfo.key, + userId, + workspaceId: context.workspaceId, + context: 'execution', + originalName: fileName, + contentType, + size: fileBuffer.length, + }, + tx + ) + await initializeWorkspaceFileSecretProvenanceInTx( + tx, + record.id, + record.contentUpdatedAt, + secretProvenance + ) + return record + }) + } const presignedUrl = await StorageService.generatePresignedDownloadUrl( fileInfo.key, @@ -130,6 +174,24 @@ export async function uploadExecutionFile( }) return userFile } catch (error) { + if (secretProvenance && uploadedKey) { + try { + await StorageService.deleteFile({ key: uploadedKey, context: 'execution' }) + if (recordedFile) { + await deleteFileMetadataByIdentity({ + id: recordedFile.id, + key: recordedFile.key, + context: 'execution', + contentUpdatedAt: recordedFile.contentUpdatedAt, + }) + } + } catch (cleanupError) { + logger.warn('Could not remove an unreturned execution file', { + key: uploadedKey, + error: getErrorMessage(cleanupError), + }) + } + } logger.error(`Failed to upload execution file ${fileName}:`, error) throw new Error(`Failed to upload file: ${getErrorMessage(error, 'Unknown error')}`) } diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts index a1b84a68099..b6bdbd17523 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts @@ -21,11 +21,16 @@ vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ })) import type { DbTransaction } from '@/lib/db/types' +import { + PROVENANCE_MAX_ENTRIES, + PROVENANCE_MAX_SERIALIZED_BYTES, +} from '@/lib/execution/provenance-limits' import { areModelSafeWorkspaceFileKeys, copyWorkspaceFileSecretProvenanceInTx, createWorkspaceFileSecretProvenanceFromRegistry, filterModelSafeWorkspaceFileAttachments, + getBoundWorkspaceFileSecretProvenance, importWorkspaceFileSecretProvenanceForModelView, importWorkspaceFileSecretProvenanceForRuntime, initializeWorkspaceFileSecretProvenanceInTx, @@ -39,6 +44,64 @@ import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secr const CONTENT_UPDATED_AT = new Date('2026-08-04T00:00:00.000Z') +describe('execution file sidecars at model boundaries', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it.each([ + { status: 'exact', version: 1, stale: false, entries: [], safe: true }, + { status: 'unknown', version: 1, stale: false, entries: [], safe: false }, + { status: 'exact', version: 1, stale: true, entries: [], safe: false }, + { status: null, version: 1, stale: false, entries: null, safe: false }, + { status: 'unknown', version: null, stale: true, entries: [], safe: true }, + { + status: 'exact', + version: 1, + stale: false, + entries: [{ name: 'KEY', encryptedValue: 'ciphertext', sourceUserId: 'writer' }], + safe: false, + }, + ])( + 'classifies execution bytes consistently: %j', + async ({ status, version, stale, entries, safe }) => { + const key = 'execution/workspace-1/workflow-1/execution-1/file.zip' + const row = { + key, + workspaceId: 'workspace-1', + context: 'execution', + fileContentUpdatedAt: CONTENT_UPDATED_AT, + secretProvenanceVersion: version, + provenanceContentUpdatedAt: stale ? new Date(0) : CONTENT_UPDATED_AT, + status, + entries, + } + for (const enforced of [false, true]) { + mockIsEnforced.mockReturnValue(enforced) + queueTableRows(workspaceFiles, [row]) + expect(await isModelSafeWorkspaceFileKey(key, { workspaceId: 'workspace-1' })).toBe(safe) + queueTableRows(workspaceFiles, [row]) + expect( + await filterModelSafeWorkspaceFileAttachments([{ id: 'invented-id', key }], { + workspaceId: 'workspace-1', + }) + ).toEqual(safe ? [{ id: 'invented-id', key }] : []) + queueTableRows(workspaceFiles, [row]) + const bound = await getBoundWorkspaceFileSecretProvenance('workspace-1', { + fileId: 'canonical-id', + key, + context: 'execution', + contentUpdatedAt: CONTENT_UPDATED_AT, + }) + expect(bound.status).toBe( + version === null || (status === 'exact' && !stale) ? 'exact' : 'unknown' + ) + } + } + ) +}) + describe('workspace file secret provenance', () => { beforeEach(() => { vi.clearAllMocks() @@ -256,7 +319,11 @@ describe('workspace file secret provenance', () => { left: 'workspaceFiles.contentUpdatedAt', right: new Date(CONTENT_UPDATED_AT.getTime() + 1), }, - { type: 'inArray', column: 'workspaceFiles.context', values: ['workspace', 'mothership'] }, + { + type: 'inArray', + column: 'workspaceFiles.context', + values: ['workspace', 'mothership', 'execution'], + }, { type: 'or', conditions: [ @@ -436,7 +503,6 @@ describe('workspace file secret provenance', () => { */ { id: 'unrecorded-id', key: 'unrecorded-key' }, { id: 'pre-marker-sidecar-id', key: 'pre-marker-sidecar-key' }, - { id: 'synthetic-execution-id', key: 'untracked-context-key' }, { id: 'legacy-id', key: 'legacy-key' }, { id: 'inline-file' }, ]) @@ -996,14 +1062,20 @@ describe('workspace file secret provenance', () => { it('merges exact byte contributors and propagates unknown classifications', () => { expect( mergeWorkspaceFileSecretProvenance( - { status: 'exact', entries: [{ name: 'A', encryptedValue: 'encrypted-a' }] }, - { status: 'exact', entries: [{ name: 'B', encryptedValue: 'encrypted-b' }] } + { + status: 'exact', + entries: [{ name: 'A', encryptedValue: 'encrypted-a', sourceUserId: 'user-1' }], + }, + { + status: 'exact', + entries: [{ name: 'B', encryptedValue: 'encrypted-b', sourceUserId: 'user-1' }], + } ) ).toEqual({ status: 'exact', entries: [ - { name: 'A', encryptedValue: 'encrypted-a' }, - { name: 'B', encryptedValue: 'encrypted-b' }, + { name: 'A', encryptedValue: 'encrypted-a', sourceUserId: 'user-1' }, + { name: 'B', encryptedValue: 'encrypted-b', sourceUserId: 'user-1' }, ], }) expect( @@ -1011,6 +1083,86 @@ describe('workspace file secret provenance', () => { ).toEqual({ status: 'unknown' }) }) + it('deduplicates only identical scoped entries across repeated contributors', () => { + const base = { + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + name: 'TOKEN', + encryptedValue: 'ciphertext', + } + const entries = [ + base, + { ...base, sourceUserId: 'user-2' }, + { ...base, sourceWorkspaceId: 'workspace-2' }, + { ...base, name: 'OTHER_TOKEN' }, + { sourceUserId: base.sourceUserId, encryptedValue: base.encryptedValue }, + { ...base, encryptedValue: 'different-ciphertext' }, + ] + const contributors = Array.from({ length: 1_000 }, () => ({ + status: 'exact' as const, + entries, + })) + + expect(mergeWorkspaceFileSecretProvenance(...contributors)).toEqual({ + status: 'exact', + entries, + }) + }) + + it('counts distinct merged entries at the actual entry boundary and refuses overflow', () => { + const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES }, (_, index) => ({ + sourceUserId: 'user-1', + encryptedValue: `ciphertext-${index}`, + })) + const full = { status: 'exact' as const, entries } + expect(mergeWorkspaceFileSecretProvenance(full, full)).toEqual(full) + expect( + mergeWorkspaceFileSecretProvenance(full, { + status: 'exact', + entries: [{ sourceUserId: 'user-1', encryptedValue: 'one-more-secret' }], + }) + ).toEqual({ status: 'unknown' }) + }) + + it('deduplicates before charging the actual byte boundary and refuses a larger union', () => { + const sourceUserId = 'user-1' + const name = 'TOKEN' + const overhead = Buffer.byteLength(sourceUserId + name, 'utf8') + const entry = { + sourceUserId, + name, + encryptedValue: 'x'.repeat(PROVENANCE_MAX_SERIALIZED_BYTES - overhead), + } + const full = { status: 'exact' as const, entries: [entry] } + expect(mergeWorkspaceFileSecretProvenance(full, full)).toEqual(full) + expect( + mergeWorkspaceFileSecretProvenance(full, { + status: 'exact', + entries: [{ sourceUserId, encryptedValue: 'one-more-secret' }], + }) + ).toEqual({ status: 'unknown' }) + expect( + mergeWorkspaceFileSecretProvenance({ + status: 'exact', + entries: [{ ...entry, encryptedValue: `${entry.encryptedValue}é` }], + }) + ).toEqual({ status: 'unknown' }) + }) + + it('stops reading entries once the merged envelope cannot be represented', () => { + const entries = [ + { sourceUserId: 'user-1', encryptedValue: 'x'.repeat(PROVENANCE_MAX_SERIALIZED_BYTES) }, + ] + Object.defineProperty(entries, 1, { + get: () => { + throw new Error('overflow must stop the merge') + }, + }) + expect(mergeWorkspaceFileSecretProvenance({ status: 'exact', entries })).toEqual({ + status: 'unknown', + }) + }) + it('does not discard known secret entries when another contributor is unrecorded', () => { const known = { status: 'exact' as const, diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts index e89f258199e..88d1b89f889 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts @@ -5,7 +5,7 @@ import { workspaceFileSecretProvenance, workspaceFiles, } from '@sim/db/schema' -import { and, eq, gte, inArray, isNull, lt, or } from 'drizzle-orm' +import { and, desc, eq, gte, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { encryptSecret } from '@/lib/core/security/encryption' import type { DbTransaction } from '@/lib/db/types' import { @@ -81,7 +81,7 @@ interface WorkspaceFileAttachmentIdentity { export interface WorkspaceFileSecretProvenanceIdentity { fileId: string key: string - context: 'workspace' | 'mothership' + context: 'workspace' | 'mothership' | 'execution' contentUpdatedAt?: Date } @@ -138,12 +138,35 @@ export function mergeWorkspaceFileSecretProvenance( : { status: 'unrecorded' } } - return { - status: 'exact', - entries: provenances.flatMap((provenance) => - provenance.status === 'exact' ? provenance.entries : [] - ), + const entries = new Map() + let bytes = 0 + for (const provenance of provenances) { + if (provenance.status !== 'exact') continue + for (const entry of provenance.entries) { + if ( + !entry.encryptedValue || + !entry.sourceUserId || + (entry.name !== undefined && entry.name.length === 0) + ) { + return { status: 'unknown' } + } + const entryBytes = exactEntryByteSize(entry) + if (entryBytes > PROVENANCE_MAX_SERIALIZED_BYTES) return { status: 'unknown' } + const key = JSON.stringify([ + entry.sourceUserId, + entry.sourceWorkspaceId ?? '', + entry.name ?? '', + entry.encryptedValue, + ]) + if (entries.has(key)) continue + bytes += entryBytes + if (entries.size >= PROVENANCE_MAX_ENTRIES || bytes > PROVENANCE_MAX_SERIALIZED_BYTES) { + return { status: 'unknown' } + } + entries.set(key, entry) + } } + return { status: 'exact', entries: [...entries.values()] } } function compareStrings(left: string, right: string): number { @@ -422,7 +445,7 @@ async function markWorkspaceFileSecretProvenanceTrackedInTx( eq(workspaceFiles.id, fileId), gte(workspaceFiles.contentUpdatedAt, contentUpdatedAt), lt(workspaceFiles.contentUpdatedAt, nextContentMillisecond), - inArray(workspaceFiles.context, ['workspace', 'mothership']), + inArray(workspaceFiles.context, ['workspace', 'mothership', 'execution']), or( isNull(workspaceFiles.secretProvenanceVersion), eq(workspaceFiles.secretProvenanceVersion, 1) @@ -859,7 +882,7 @@ export async function getBoundWorkspaceFileSecretProvenanceByMetadata( * absence this covers. Closing the surface again is a matter of naming it in * `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES`. */ -function mayReadUnrecordedWorkspaceFile( +export function mayReadUnrecordedWorkspaceFile( workspaceId: string | undefined, count = 1, actorUserId?: string @@ -1017,7 +1040,8 @@ export async function importWorkspaceFileSecretProvenanceForRuntime(args: { /** * Removes model attachments whose canonical workspace-file record is tainted or unknown. * Missing legacy records remain compatible; persisted records are classified by their unique - * active storage-key binding and private provenance row. Attachment ids are deliberately ignored: + * storage-key binding (active first, newest archived execution revision otherwise) and private + * provenance row. Attachment ids are deliberately ignored: * older persisted workflows omit them and file normalization may synthesize a runtime-only id. * This classification is not file authorization; callers still enforce storage access before * reading bytes or issuing a provider URL. @@ -1051,7 +1075,13 @@ export async function filterModelSafeWorkspaceFileAttachments< if (typeof attachment.key !== 'string' || attachment.key.length === 0) return true const row = rowByKey.get(attachment.key) if (!row) return true - if (row.context !== 'workspace' && row.context !== 'mothership') return true + if ( + row.context !== 'workspace' && + row.context !== 'mothership' && + row.context !== 'execution' + ) { + return true + } const classification = classifyModelSafeWorkspaceFileRow(row, options.workspaceId) if (classification === 'safe') return true if (classification === 'unsafe') { @@ -1098,7 +1128,7 @@ async function loadModelSafeWorkspaceFileRows( keys: readonly string[] ): Promise { return db - .select({ + .selectDistinctOn([workspaceFiles.key], { key: workspaceFiles.key, workspaceId: workspaceFiles.workspaceId, context: workspaceFiles.context, @@ -1113,7 +1143,18 @@ async function loadModelSafeWorkspaceFileRows( workspaceFileSecretProvenance, eq(workspaceFileSecretProvenance.fileId, workspaceFiles.id) ) - .where(and(inArray(workspaceFiles.key, [...keys]), isNull(workspaceFiles.deletedAt))) + .where( + and( + inArray(workspaceFiles.key, [...keys]), + or(isNull(workspaceFiles.deletedAt), eq(workspaceFiles.context, 'execution')) + ) + ) + .orderBy( + workspaceFiles.key, + sql`${workspaceFiles.deletedAt} IS NULL DESC`, + desc(workspaceFiles.contentUpdatedAt), + workspaceFiles.id + ) } /** @@ -1131,8 +1172,8 @@ export async function isModelSafeWorkspaceFileKey( /** * Batch variant for server-authorized storage keys crossing the same model boundary. Missing keys - * and non-workspace contexts retain their legacy/raw behavior; canonical workspace and mothership - * rows are accepted only when every current content version has exact-empty provenance. + * retain their legacy behavior; tracked workspace, mothership, and execution files must satisfy + * the same classification before their bytes leave private storage. */ export async function areModelSafeWorkspaceFileKeys( keys: readonly string[], @@ -1148,7 +1189,13 @@ export async function areModelSafeWorkspaceFileKeys( let unrecorded = 0 for (const row of rows) { - if (row.context !== 'workspace' && row.context !== 'mothership') continue + if ( + row.context !== 'workspace' && + row.context !== 'mothership' && + row.context !== 'execution' + ) { + continue + } const classification = classifyModelSafeWorkspaceFileRow(row, options.workspaceId) if (classification === 'unsafe') { return refuseWorkspaceFileProvenance( diff --git a/apps/sim/lib/uploads/server/metadata.ts b/apps/sim/lib/uploads/server/metadata.ts index f0d9661fd6d..97ab3565e38 100644 --- a/apps/sim/lib/uploads/server/metadata.ts +++ b/apps/sim/lib/uploads/server/metadata.ts @@ -3,7 +3,7 @@ import { withInsertColumns } from '@sim/db/insert-columns' import { type WorkspaceFileRow, workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, isNotNull, isNull, sql } from 'drizzle-orm' +import { and, desc, eq, inArray, isNotNull, isNull, sql } from 'drizzle-orm' import type { DbOrTx, DbTransaction } from '@/lib/db/types' import { getWorkspaceFileSize, @@ -424,18 +424,31 @@ export async function resolveStoredFileContext(key: string): Promise = db, - options?: { lock?: 'share' } + executor: Pick = db, + options?: { lock?: 'share'; includeDeleted?: false } | { lock?: never; includeDeleted: true } ): Promise { if (keys.length === 0) { return [] } + if (options?.includeDeleted) { + return executor + .selectDistinctOn([workspaceFiles.key], workspaceFileColumns) + .from(workspaceFiles) + .where(and(inArray(workspaceFiles.key, keys), eq(workspaceFiles.context, context))) + .orderBy( + workspaceFiles.key, + sql`${workspaceFiles.deletedAt} IS NULL DESC`, + desc(workspaceFiles.contentUpdatedAt), + workspaceFiles.id + ) + } const query = executor .select(workspaceFileColumns) .from(workspaceFiles) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.test.ts b/apps/sim/lib/uploads/utils/file-utils.server.test.ts index 91670aeeb25..f3463527fa8 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.test.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.test.ts @@ -3,13 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDownloadFile, mockParseWorkspaceFileKey, mockResolveServableDocBytes } = vi.hoisted( - () => ({ +const { mockDownloadFile, mockParseWorkspaceFileKey, mockResolveServableDocBytes, mockRenderPage } = + vi.hoisted(() => ({ mockDownloadFile: vi.fn(), mockParseWorkspaceFileKey: vi.fn(), mockResolveServableDocBytes: vi.fn(), - }) -) + mockRenderPage: vi.fn(), + })) vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFile: mockDownloadFile, @@ -28,6 +28,10 @@ vi.mock('@/lib/copilot/tools/server/files/doc-compile', () => ({ resolveServableDocBytes: mockResolveServableDocBytes, })) +vi.mock('@/lib/workspace-files/page-document.server', () => ({ + renderSimPageDocumentWithContributors: mockRenderPage, +})) + vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: vi.fn(), })) @@ -220,3 +224,43 @@ describe('downloadServableFilesWithinBudget', () => { expect(mockDownloadFile).toHaveBeenCalledTimes(1) }) }) + +describe('servable page provenance', () => { + it('preserves the inlined image identity for execution-stored pages', async () => { + const workspaceId = '2f1d8c3e-5b6a-4c7d-8e9f-0a1b2c3d4e5f' + const contributor = { + fileId: 'image-file', + key: `workspace/${workspaceId}/image.png`, + context: 'workspace' as const, + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), + } + mockParseWorkspaceFileKey.mockReturnValue(null) + mockDownloadFile.mockResolvedValue(Buffer.from('---\ntitle: Example\n---\nPage body')) + mockRenderPage.mockResolvedValue({ + html: 'rendered image', + contributingFiles: [contributor], + }) + + const rendered = await downloadServableFileFromStorage( + { + id: 'page-file', + name: 'page.html', + key: `execution/${workspaceId}/3f2e9d4c-6a7b-4d8e-9f0a-1b2c3d4e5f6a/4a3b2c1d-7e8f-4a9b-8c0d-1e2f3a4b5c6d/page.html`, + url: '', + type: 'text/x-sim-page', + size: 100, + context: 'execution', + }, + 'request', + createLogger('test'), + { maxBytes: 1024 } + ) + + expect(mockRenderPage).toHaveBeenCalledWith(expect.any(String), { workspaceId }) + expect(rendered).toEqual({ + buffer: Buffer.from('rendered image'), + contentType: 'text/html', + contributingFiles: [contributor], + }) + }) +}) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index b78bff2b00b..625dcddc262 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -36,7 +36,7 @@ import { resolveTrustedFileContext, } from '@/lib/uploads/utils/file-utils' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' -import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' +import { renderSimPageDocumentWithContributors } from '@/lib/workspace-files/page-document.server' import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' @@ -461,16 +461,20 @@ export async function downloadServableFileFromStorage( const text = buffer.toString('utf8') if (isSimPageSource(text)) { const workspaceId = userFile.key - ? (parseWorkspaceFileKey(userFile.key) ?? undefined) + ? (parseWorkspaceFileKey(userFile.key) ?? + extractWorkspaceIdFromExecutionKey(userFile.key) ?? + undefined) : undefined - const rendered = Buffer.from( - await renderSimPageDocumentWithAssets(text, { workspaceId }), - 'utf8' - ) + const page = await renderSimPageDocumentWithContributors(text, { workspaceId }) + const rendered = Buffer.from(page.html, 'utf8') // Rendering inlines referenced assets, so a source well under the ceiling can // resolve to a document well over it. assertKnownSizeWithinLimit(rendered.length, options.maxBytes, 'servable page render') - return { buffer: rendered, contentType: 'text/html' } + return { + buffer: rendered, + contentType: 'text/html', + contributingFiles: page.contributingFiles, + } } } diff --git a/apps/sim/lib/workspace-files/page-document.server.test.ts b/apps/sim/lib/workspace-files/page-document.server.test.ts index 14375b70a16..5eba3bbd766 100644 --- a/apps/sim/lib/workspace-files/page-document.server.test.ts +++ b/apps/sim/lib/workspace-files/page-document.server.test.ts @@ -21,7 +21,10 @@ vi.mock('@/lib/workspace-files/page-document', () => ({ renderSimPageDocument: mockRenderSimPageDocument, })) -import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' +import { + renderSimPageDocumentWithAssets, + renderSimPageDocumentWithContributors, +} from '@/lib/workspace-files/page-document.server' const WORKSPACE_ID = 'ws-1' const MB = 1024 * 1024 @@ -35,6 +38,7 @@ function imageRecord(id: string, size: number) { contentType: 'image/png', size, sizeBytes: size, + contentUpdatedAt: new Date('2026-01-01T00:00:00Z'), } } @@ -119,3 +123,73 @@ describe('renderSimPageDocumentWithAssets memory bounds', () => { expect(html).toContain('src="/api/files/view/theirs"') }) }) + +describe('rendered page contributors', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('reports only the canonical revisions whose bytes were embedded', async () => { + mockRenderSimPageDocument.mockReturnValue( + documentReferencing(['mine', 'failed', 'foreign', 'missing', 'mine']) + ) + const record = imageRecord('mine', 5) + mockGetFileMetadataById.mockImplementation(async (id: string) => { + if (id === 'missing') return null + if (id === 'foreign') return { ...imageRecord(id, 5), workspaceId: 'other-workspace' } + return id === 'mine' ? record : imageRecord(id, 5) + }) + mockDownloadFile.mockImplementation(async ({ key }: { key: string }) => { + if (key.includes('failed')) throw new Error('unavailable') + return Buffer.from('image') + }) + + const rendered = await renderSimPageDocumentWithContributors('source', { + workspaceId: WORKSPACE_ID, + }) + + expect(rendered.contributingFiles).toEqual([ + { + fileId: record.id, + key: record.key, + context: 'workspace', + contentUpdatedAt: record.contentUpdatedAt, + }, + ]) + expect(rendered.html).toContain('data:image/png;base64,aW1hZ2U=') + expect(rendered.html).toContain('/api/files/view/failed') + expect(rendered.html).toContain('/api/files/view/foreign') + expect(mockGetFileMetadataById).toHaveBeenCalledTimes(4) + }) + + it('bounds metadata reads for missing images', async () => { + mockRenderSimPageDocument.mockReturnValue( + documentReferencing(Array.from({ length: 300 }, (_, i) => `missing-${i}`)) + ) + mockGetFileMetadataById.mockResolvedValue(null) + + const rendered = await renderSimPageDocumentWithContributors('source', { + workspaceId: WORKSPACE_ID, + }) + + expect(mockGetFileMetadataById).toHaveBeenCalledTimes(256) + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(rendered.contributingFiles).toEqual([]) + }) + + it('charges repeated image occurrences against the rendered byte budget', async () => { + const source = documentReferencing(Array(12).fill('image')) + mockRenderSimPageDocument.mockReturnValue(source) + mockGetFileMetadataById.mockResolvedValue(imageRecord('image', 8 * MB)) + mockDownloadFile.mockResolvedValue(Buffer.alloc(8 * MB)) + + const rendered = await renderSimPageDocumentWithContributors('source', { + workspaceId: WORKSPACE_ID, + }) + + expect(mockDownloadFile).toHaveBeenCalledTimes(1) + expect(rendered.html.length).toBeLessThanOrEqual(source.length + Math.ceil((32 * MB * 4) / 3)) + expect(rendered.html).toContain('/api/files/view/image') + expect(rendered.contributingFiles).toHaveLength(1) + }) +}) diff --git a/apps/sim/lib/workspace-files/page-document.server.ts b/apps/sim/lib/workspace-files/page-document.server.ts index 4c4954e33dc..7abcdfa74d4 100644 --- a/apps/sim/lib/workspace-files/page-document.server.ts +++ b/apps/sim/lib/workspace-files/page-document.server.ts @@ -1,3 +1,4 @@ +import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { downloadFile } from '@/lib/uploads/core/storage-service' import { getFileMetadataById } from '@/lib/uploads/server/metadata' import { renderSimPageDocument } from '@/lib/workspace-files/page-document' @@ -12,6 +13,9 @@ const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 */ const MAX_INLINE_TOTAL_BYTES = 32 * 1024 * 1024 +/** Bounds metadata reads even when the page references many missing or empty images. */ +const MAX_INLINE_IMAGE_REFERENCES = 256 + const IMAGE_SRC = /src="[^"]*\/api\/files\/view\/([^"]+)"/g /** @@ -27,30 +31,32 @@ export async function renderSimPageDocumentWithAssets( source: string, options: { workspaceId?: string } ): Promise { - const documentHtml = renderSimPageDocument(source, options) - const ids = [...new Set([...documentHtml.matchAll(IMAGE_SRC)].map((match) => match[1]))] - if (ids.length === 0 || !options.workspaceId) return documentHtml + return (await renderSimPageDocumentWithContributors(source, options)).html +} - const candidates = await Promise.all( - ids.map(async (id) => { - const record = await getFileMetadataById(id).catch(() => null) - if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) - return null - return { id, record } - }) - ) +/** Servable page bytes and the exact stored image revisions actually embedded in them. */ +export async function renderSimPageDocumentWithContributors( + source: string, + options: { workspaceId?: string } +): Promise<{ html: string; contributingFiles: readonly WorkspaceFileSecretProvenanceIdentity[] }> { + const documentHtml = renderSimPageDocument(source, options) + if (!options.workspaceId) return { html: documentHtml, contributingFiles: [] } - // One image at a time, charged against the budget by what each download actually - // delivered. Fetching them concurrently made the peak the sum of every image rather - // than the largest one, and the ceiling on the finished document could only observe - // that after the fact. Each download is given whatever the budget has left, so an - // image that does not fit is refused by the read itself instead of after it lands. - const inlined = new Map() + const visited = new Set() + const inlined = new Map< + string, + { dataUri: string; identity: WorkspaceFileSecretProvenanceIdentity } + >() let remaining = MAX_INLINE_TOTAL_BYTES - for (const candidate of candidates) { - if (!candidate) continue - if (remaining === 0) break - const { id, record } = candidate + for (const match of documentHtml.matchAll(IMAGE_SRC)) { + const id = match[1] + if (visited.has(id)) continue + if (remaining === 0 || visited.size >= MAX_INLINE_IMAGE_REFERENCES) break + visited.add(id) + const record = await getFileMetadataById(id).catch(() => null) + if (!record || record.context !== 'workspace' || record.workspaceId !== options.workspaceId) { + continue + } try { const bytes = await downloadFile({ key: record.key, @@ -61,14 +67,28 @@ export async function renderSimPageDocumentWithAssets( const mime = record.contentType?.startsWith('image/') ? record.contentType : 'application/octet-stream' - inlined.set(id, `data:${mime};base64,${bytes.toString('base64')}`) + inlined.set(id, { + dataUri: `data:${mime};base64,${bytes.toString('base64')}`, + identity: { + fileId: record.id, + key: record.key, + context: 'workspace', + contentUpdatedAt: record.contentUpdatedAt, + }, + }) } catch { - // A missing, unreadable or too-large image keeps its URL reference. + /** A missing, unreadable or too-large image keeps its URL reference. */ } } - if (inlined.size === 0) return documentHtml - return documentHtml.replace(IMAGE_SRC, (match, id: string) => { - const dataUri = inlined.get(id) - return dataUri ? `src="${dataUri}"` : match + /** Charge each occurrence: repeating one image must not multiply the rendered byte budget. */ + let remainingEncodedBytes = Math.ceil((MAX_INLINE_TOTAL_BYTES * 4) / 3) + const contributors = new Map() + const html = documentHtml.replace(IMAGE_SRC, (match, id: string) => { + const image = inlined.get(id) + if (!image || image.dataUri.length > remainingEncodedBytes) return match + remainingEncodedBytes -= image.dataUri.length + contributors.set(id, image.identity) + return `src="${image.dataUri}"` }) + return { html, contributingFiles: [...contributors.values()] } } diff --git a/apps/sim/tools/file/parser.test.ts b/apps/sim/tools/file/parser.test.ts index 8a0e3e1ac91..18f794e0228 100644 --- a/apps/sim/tools/file/parser.test.ts +++ b/apps/sim/tools/file/parser.test.ts @@ -2,9 +2,20 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { fileFetchTool, fileParserTool, fileParserV3Tool } from '@/tools/file/parser' +import { + fileFetchTool, + fileParserTool, + fileParserV2Tool, + fileParserV3Tool, +} from '@/tools/file/parser' describe('fileParserTool', () => { + it.each([fileFetchTool, fileParserTool, fileParserV2Tool, fileParserV3Tool])( + '$id negotiates stored source provenance before exposing parsed content', + (tool) => { + expect(tool.operation.secretProvenance?.response).toEqual({ incomplete: 'reject' }) + } + ) it('maps the public File Fetch URL to the internal parser path', () => { expect( fileFetchTool.operation.input({ diff --git a/apps/sim/tools/file/parser.ts b/apps/sim/tools/file/parser.ts index fb4d8cf869f..a17a1c9a03c 100644 --- a/apps/sim/tools/file/parser.ts +++ b/apps/sim/tools/file/parser.ts @@ -197,6 +197,7 @@ export const fileParserTool: InternalToolConfig { logger.info('Request parameters received by tool body:', params) @@ -384,6 +385,7 @@ export const fileFetchTool: InternalToolConfig fileParserTool.operation.input({ ...params, diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 1cb84748caa..7a84d491bef 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -1562,6 +1562,66 @@ describe('executeTool Function', () => { ]) }) + it.each([true, false])( + 'carries File Fetch lineage into later durable values when complete=%s', + async (complete) => { + const scope = { userId: 'user-1', workspaceId: 'workspace-1' } + const registry = new ResolvedSecretTraceRegistry([], scope) + const entry = { name: 'API_KEY', encryptedValue: 'encrypted-value' } + encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: 'secret-value' }) + mockExecuteInternalToolOperation.mockResolvedValueOnce( + Response.json( + { + success: true, + output: { + content: 'secret-value', + name: 'report.txt', + fileType: 'text/plain', + size: 12, + binary: false, + }, + __resolvedSecretTraceProvenance: { + version: 1, + complete, + entries: complete ? [entry] : [], + scope, + }, + }, + { headers: { 'x-sim-private-tool-metadata': 'resolved-secret-provenance-v1' } } + ) + ) + + const result = await executeTool( + 'file_fetch', + { fileUrl: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/report.txt' }, + { + executionContext: createToolExecutionContext(scope), + resolvedSecretTraceRegistry: registry, + } + ) + + expect(result).toMatchObject({ + success: true, + output: { combinedContent: 'secret-value' }, + }) + expect(JSON.stringify(result)).not.toContain('__resolvedSecretTraceProvenance') + expect( + mockExecuteInternalToolOperation.mock.calls[0]?.[0].headers.get( + 'x-sim-request-private-tool-metadata' + ) + ).toBe('resolved-secret-provenance-v1') + for (const durableValue of [ + { 'column-id': 'secret-value' }, + { role: 'assistant', content: 'secret-value' }, + ]) { + expect(registry.exportCommittedProvenanceForValue(durableValue)).toMatchObject({ + complete, + entries: complete ? [entry] : [], + }) + } + } + ) + it.each([ { name: 'table propagate policy',