Skip to content

Commit aeb7eae

Browse files
authored
feat(model): sim auto model (#6103)
* Add sim-auto: automatic model routing for the agent block (hosted) - 'Auto' model option (Sim wordmark icon, hosted-only, new-block default; runtime fallback for unset models stays claude-sonnet-5) - Resolver (lib/model-router): classifies each execution via mothership's /api/model-router and routes over two ladders — text: fireworks/glm-5.2 -> fireworks/kimi-k3 (new static hosted Fireworks catalog entries on the platform FIREWORKS_API_KEY); attachments: claude-haiku-4-5 -> gpt-5.5 (Fireworks OSS endpoints reject images). Trivial tasks skip the router; 5-min decision cache; 2s timeout; never fails the workflow - Hidden identity preamble on every auto run (English by default, don't volunteer the underlying model) - Fireworks executor: wire-name map for catalog ids (glm-5.2 -> glm-5p2), pricing keyed on the full catalog id - Billing: routing cost applied to non-streaming output cost only when mothership marks the call billable (bill-model-router flag, default off) - sim-auto special-cases: API-key condition, serialization tool lookup, edit-workflow validation (hosted), VFS model options projection * auto model * agent auto model * fix lint
1 parent 6647808 commit aeb7eae

28 files changed

Lines changed: 1391 additions & 33 deletions

File tree

Lines changed: 3 additions & 0 deletions
Loading
23.5 KB
Loading
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"fill": {
3+
"solid": "srgb:1.00000,1.00000,1.00000,1.00000"
4+
},
5+
"groups": [
6+
{
7+
"layers": [
8+
{
9+
"image-name": "logo.png",
10+
"is-glass": false,
11+
"name": "Sim"
12+
},
13+
{
14+
"image-name": "border.svg",
15+
"is-glass": false,
16+
"name": "Dev Border"
17+
}
18+
],
19+
"shadow": {
20+
"kind": "neutral",
21+
"opacity": 0
22+
},
23+
"specular": false,
24+
"translucency": {
25+
"enabled": false,
26+
"value": 0
27+
}
28+
}
29+
],
30+
"supported-platforms": {
31+
"squares": ["macOS"]
32+
}
33+
}

apps/sim/.env.example

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,11 @@ API_ENCRYPTION_KEY=your_api_encryption_key # Use `openssl rand -hex 32` to gener
6363
# VLLM_API_KEY= # Optional bearer token if your vLLM instance requires auth
6464
# LITELLM_BASE_URL=http://localhost:4000 # Base URL for your LiteLLM proxy (OpenAI-compatible)
6565
# LITELLM_API_KEY= # Optional bearer token if your LiteLLM proxy requires auth
66-
# FIREWORKS_API_KEY= # Optional Fireworks AI API key for model listing
66+
# NEXT_PUBLIC_FORCE_HOSTED=true # Dev only: treat this instance as hosted Sim (sim-auto pool, platform keys); ignored in production builds
67+
# FIREWORKS_API_KEY= # Optional Fireworks AI API key for model listing and inference
68+
# FIREWORKS_API_KEY_1= # Optional Fireworks API key for rotation (hosted deployments)
69+
# FIREWORKS_API_KEY_2= # Additional Fireworks API key for load balancing
70+
# FIREWORKS_API_KEY_3= # Additional Fireworks API key for load balancing
6771
# NEXT_PUBLIC_BEDROCK_DEFAULT_CREDENTIALS=true # Set when using AWS default credential chain (IAM roles, ECS task roles, IRSA). Hides credential fields in Agent block UI.
6872
# AZURE_OPENAI_ENDPOINT= # Azure OpenAI endpoint (hides field in UI when set alongside NEXT_PUBLIC_AZURE_CONFIGURED)
6973
# AZURE_OPENAI_API_KEY= # Azure OpenAI API key

apps/sim/app/api/providers/fireworks/models/route.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { validationErrorResponse } from '@/lib/api/server'
1010
import { getBYOKKey } from '@/lib/api-key/byok'
1111
import { getSession } from '@/lib/auth'
1212
import { env } from '@/lib/core/config/env'
13+
import { isHosted } from '@/lib/core/config/env-flags'
1314
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
1415
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
1516
import { filterBlacklistedModels, isProviderBlacklisted } from '@/providers/utils'
@@ -54,7 +55,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
5455
}
5556
}
5657

57-
if (!apiKey) {
58+
/**
59+
* On hosted Sim the platform key is the sim-auto pool's inference key, not a
60+
* workspace credential: enumerating the whole Fireworks catalog from it would
61+
* offer every serverless model as selectable when no key can actually run it
62+
* (`getApiKeyWithBYOK` serves the platform key to catalog models only).
63+
*/
64+
if (!apiKey && !isHosted) {
5865
apiKey = env.FIREWORKS_API_KEY
5966
}
6067

apps/sim/blocks/blocks/agent.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
getReasoningEffortValuesForModel,
2121
getThinkingLevelsForModel,
2222
getVerbosityValuesForModel,
23+
isAutoModel,
2324
supportsTemperature,
2425
} from '@/providers/models'
2526
import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
@@ -522,7 +523,13 @@ Return ONLY the JSON array.`,
522523
if (!model) {
523524
throw new Error('No model selected')
524525
}
525-
const tool = getBaseModelProviders()[model]
526+
// sim-auto resolves to a concrete pool model at execution time, where
527+
// the agent handler derives the provider from the resolved model and
528+
// never reads this serialized value. Serialization still needs the
529+
// same provider-id shape every other model stores, so look up the
530+
// runtime fallback model's provider.
531+
const lookupModel = isAutoModel(model) ? 'claude-sonnet-5' : model
532+
const tool = getBaseModelProviders()[lookupModel]
526533
if (!tool) {
527534
throw new Error(`Invalid model selected: ${model}`)
528535
}

apps/sim/blocks/utils.test.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ vi.mock('@/providers/models', () => ({
4040
getProviderModels: mockGetProviderModels,
4141
getProviderIcon: mockGetProviderIcon,
4242
getBaseModelProviders: mockGetBaseModelProviders,
43+
SIM_AUTO_MODEL_ID: 'sim-auto',
44+
isAutoModel: (model: string) => model.trim().toLowerCase() === 'sim-auto',
4345
}))
4446

4547
vi.mock('@/providers/utils', () => ({

apps/sim/blocks/utils.ts

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { toError } from '@sim/utils/errors'
2+
import { SimAutoIcon } from '@/components/icons'
23
import {
34
isAzureConfigured,
45
isCohereConfigured,
@@ -14,7 +15,9 @@ import {
1415
getModelSunsetStatus,
1516
getProviderIcon,
1617
getProviderModels,
18+
isAutoModel,
1719
orderModelIdsByReleaseDate,
20+
SIM_AUTO_MODEL_ID,
1821
} from '@/providers/models'
1922
import { isPiSupportedModel } from '@/providers/pi-providers'
2023
import { getProviderFromModel } from '@/providers/utils'
@@ -75,12 +78,21 @@ export function getModelOptions() {
7578
])
7679
)
7780

78-
return allModels
81+
const options = allModels
7982
.filter((model) => getModelSunsetStatus(model) !== 'deprecated')
8083
.map((model) => {
8184
const icon = getProviderIcon(model)
8285
return { label: model, id: model, ...(icon && { icon }) }
8386
})
87+
88+
// Hosted-only automatic model. Deliberately LAST in the list (limited
89+
// visibility for the initial release): available to anyone who scrolls or
90+
// searches for it, but never the first thing the dropdown offers.
91+
if (isHosted) {
92+
options.push({ label: 'Auto', id: SIM_AUTO_MODEL_ID, icon: SimAutoIcon })
93+
}
94+
95+
return options
8496
}
8597

8698
/**
@@ -183,6 +195,11 @@ function shouldRequireApiKeyForModel(model: string): boolean {
183195
const normalizedModel = model.trim().toLowerCase()
184196
if (!normalizedModel) return false
185197

198+
// On hosted Sim the auto pseudo-model resolves server-side to a hosted pool
199+
// model. On self-hosted it exists only via imported workflows and always
200+
// falls back to the default Anthropic model, so the key field must show.
201+
if (isAutoModel(normalizedModel)) return !isHosted
202+
186203
if (isHosted) {
187204
const hostedModels = getHostedModels()
188205
if (hostedModels.some((m) => m.toLowerCase() === normalizedModel)) return false

apps/sim/components/icons.tsx

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7558,6 +7558,31 @@ export function SixtyfourIcon(props: SVGProps<SVGSVGElement>) {
75587558
)
75597559
}
75607560

7561+
/**
7562+
* The "sim" brand wordmark (v1.0 brand guide simLogotype paths — the same mark
7563+
* the navbar/login header renders), inked with the theme-adaptive
7564+
* `--text-body`. Used as the icon for the Auto model option; the wide viewBox
7565+
* letterboxes itself inside square icon slots.
7566+
*/
7567+
export function SimAutoIcon(props: SVGProps<SVGSVGElement>) {
7568+
return (
7569+
<svg
7570+
{...props}
7571+
viewBox='0 0 441 212'
7572+
fill='none'
7573+
xmlns='http://www.w3.org/2000/svg'
7574+
aria-hidden='true'
7575+
>
7576+
<g fill='var(--text-body)'>
7577+
<path d='M0 160.9H29.51C29.51 169.08 32.46 175.61 38.37 180.48C44.27 185.12 52.25 187.44 62.31 187.44C73.24 187.44 81.65 185.34 87.56 181.14C93.46 176.71 96.41 170.85 96.41 163.55C96.41 158.24 94.77 153.82 91.49 150.28C88.43 146.74 82.75 143.86 74.44 141.65L46.24 135.01C32.03 131.47 21.42 126.05 14.43 118.75C7.65 111.45 4.26 101.83 4.26 89.88C4.26 79.93 6.78 71.3 11.81 64C17.05 56.7 24.16 51.06 33.12 47.08C42.3 43.09 52.8 41.1 64.6 41.1C76.41 41.1 86.57 43.2 95.1 47.41C103.84 51.61 110.62 57.47 115.43 64.99C120.46 72.52 123.08 81.48 123.3 91.87H93.79C93.57 83.47 90.84 76.94 85.59 72.3C80.34 67.65 73.02 65.33 63.62 65.33C54 65.33 46.57 67.43 41.32 71.63C36.07 75.83 33.45 81.59 33.45 88.89C33.45 99.73 41.32 107.14 57.06 111.12L85.26 118.09C98.81 121.19 108.98 126.28 115.76 133.35C122.53 140.21 125.92 149.61 125.92 161.56C125.92 171.74 123.19 180.7 117.73 188.44C112.26 195.96 104.72 201.82 95.1 206.03C85.7 210.01 74.55 212 61.65 212C42.85 212 27.87 207.35 16.72 198.06C5.57 188.77 0 176.38 0 160.9Z' />
7578+
<path d='M232.8 212H202.13L202.13 49.76H229.54V77.39C232.8 68.34 239.11 60.66 247.81 54.7C256.73 48.52 267.5 45.43 280.12 45.43C294.26 45.43 306.01 49.29 315.36 57.02C324.72 64.75 330.81 75.01 333.64 87.82H328.09C330.27 75.01 336.25 64.75 346.04 57.02C355.83 49.29 367.9 45.43 382.26 45.43C400.54 45.43 414.89 50.84 425.34 61.66C435.78 72.47 441 87.26 441 106.03V212H410.98V113.65C410.98 100.84 407.71 91.02 401.19 84.17C394.88 77.11 386.29 73.58 375.41 73.58C367.79 73.58 361.05 75.34 355.17 78.88C349.52 82.19 345.06 87.04 341.8 93.45C338.53 99.85 336.9 107.36 336.9 115.97V212H306.55V113.32C306.55 100.51 303.4 90.8 297.09 84.17C290.78 77.33 282.19 73.91 271.31 73.91C263.69 73.91 256.95 75.67 251.08 79.21C245.42 82.52 240.96 87.38 237.7 93.78C234.43 99.96 232.8 107.36 232.8 115.97V212Z' />
7579+
<path d='M184.83 20.55C184.83 31.9 175.64 41.1 164.29 41.1C152.95 41.1 143.76 31.9 143.76 20.55C143.76 9.2 152.95 0 164.29 0C175.64 0 184.83 9.2 184.83 20.55Z' />
7580+
<path d='M179.43 212H149.16V49.76C153.76 51.91 158.88 53.12 164.29 53.12C169.7 53.12 174.83 51.91 179.43 49.76V212Z' />
7581+
</g>
7582+
</svg>
7583+
)
7584+
}
7585+
75617586
export function SimTriggerIcon(props: SVGProps<SVGSVGElement>) {
75627587
return (
75637588
<svg

apps/sim/executor/handlers/agent/agent-handler.test.ts

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ import {
1717
vi,
1818
} from 'vitest'
1919
import { getAllBlocks } from '@/blocks'
20-
import { BlockType, isMcpTool } from '@/executor/constants'
20+
import { AGENT, BlockType, isMcpTool } from '@/executor/constants'
2121
import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler'
2222
import type { ExecutionContext, StreamingExecution } from '@/executor/types'
2323
import { executeProviderRequest } from '@/providers'
24+
import { installStreamingCostPolicy } from '@/providers/cost-policy'
25+
import { SIM_AUTO_MODEL_ID } from '@/providers/models'
2426
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
2527
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
2628
import { executeTool } from '@/tools'
@@ -275,6 +277,127 @@ describe('AgentBlockHandler', () => {
275277
expect(result).toEqual(expectedOutput)
276278
})
277279

280+
it('reports a sim-auto run under the sim-auto identity, not the model that served it', async () => {
281+
mockExecuteProviderRequest.mockResolvedValue({
282+
content: 'Mocked response content',
283+
model: AGENT.DEFAULT_MODEL,
284+
tokens: { input: 10, output: 20, total: 30 },
285+
toolCalls: [],
286+
cost: { input: 0.001, output: 0.002, total: 0.003 },
287+
timing: {
288+
total: 100,
289+
timeSegments: [
290+
{ type: 'model', name: AGENT.DEFAULT_MODEL, provider: 'anthropic', duration: 100 },
291+
],
292+
},
293+
})
294+
295+
const result = (await handler.execute(mockContext, mockBlock, {
296+
model: SIM_AUTO_MODEL_ID,
297+
userPrompt: 'Hello!',
298+
})) as {
299+
model: string
300+
cost: unknown
301+
tokens: unknown
302+
providerTiming: { timeSegments: Array<{ name?: string; provider?: string }> }
303+
}
304+
305+
expect(result.model).toBe(SIM_AUTO_MODEL_ID)
306+
expect(result.providerTiming.timeSegments[0].name).toBe(SIM_AUTO_MODEL_ID)
307+
expect(result.providerTiming.timeSegments[0].provider).toBeUndefined()
308+
// Only the label changes: tokens and the already-priced cost are untouched.
309+
expect(result.tokens).toEqual({ input: 10, output: 20, total: 30 })
310+
expect(result.cost).toEqual({ input: 0.001, output: 0.002, total: 0.003 })
311+
})
312+
313+
/** Reaches the private signal builder; routing depends on nothing else. */
314+
const buildAutoRoutingSignalsFor = (inputs: Record<string, unknown>) =>
315+
(
316+
handler as unknown as {
317+
buildAutoRoutingSignals: (i: unknown, rf: unknown) => { mediaKind: string }
318+
}
319+
).buildAutoRoutingSignals(inputs, undefined)
320+
321+
const png = { id: 'f1', type: 'image/png' }
322+
const pdf = { id: 'f2', type: 'application/pdf' }
323+
324+
it('reports no media when neither the files input nor any message carries one', async () => {
325+
const signals = buildAutoRoutingSignalsFor({
326+
messages: [{ role: 'user' as const, content: 'Summarize this text' }],
327+
})
328+
329+
expect(signals.mediaKind).toBe('none')
330+
})
331+
332+
it('detects media carried on inbound messages, not just the files input', async () => {
333+
const signals = buildAutoRoutingSignalsFor({
334+
messages: [{ role: 'user' as const, content: 'What is in this image?', files: [png] }],
335+
})
336+
337+
expect(signals.mediaKind).toBe('image')
338+
})
339+
340+
it('classifies an all-image attachment set as image', async () => {
341+
expect(buildAutoRoutingSignalsFor({ files: [png, png] }).mediaKind).toBe('image')
342+
})
343+
344+
it('classifies a mixed image + document set as file', async () => {
345+
expect(buildAutoRoutingSignalsFor({ files: [png, pdf] }).mediaKind).toBe('file')
346+
})
347+
348+
it('treats an unknown MIME type as file rather than assuming it is an image', async () => {
349+
expect(buildAutoRoutingSignalsFor({ files: [{ id: 'f3' }] }).mediaKind).toBe('file')
350+
})
351+
352+
it('overlays the routing charge on a streaming cost written after the fact', async () => {
353+
// Mirrors the real streaming shape: the policy accessor is installed at
354+
// provider-return time, the drain writes the final cost long after the
355+
// handler returned, and consumers read it at log time.
356+
const output: Record<string, unknown> = { cost: { input: 0, output: 0, total: 0 } }
357+
installStreamingCostPolicy(output as never, { billable: true, multiplier: 1 })
358+
const streaming = { stream: new ReadableStream(), execution: { output } }
359+
360+
;(
361+
handler as unknown as { applyRoutingCost: (r: unknown, c: number) => void }
362+
).applyRoutingCost(streaming, 0.002)
363+
364+
// The drain settles the model cost afterwards.
365+
366+
;(output as { cost: unknown }).cost = { input: 0.01, output: 0.02, total: 0.03 }
367+
368+
expect(output.cost).toEqual({
369+
input: 0.01,
370+
output: 0.02,
371+
total: expect.closeTo(0.032, 10),
372+
routing: 0.002,
373+
})
374+
})
375+
376+
it('adds the routing charge to a settled non-streaming cost', async () => {
377+
const result: Record<string, unknown> = { cost: { input: 0.01, output: 0.02, total: 0.03 } }
378+
379+
;(
380+
handler as unknown as { applyRoutingCost: (r: unknown, c: number) => void }
381+
).applyRoutingCost(result, 0.002)
382+
383+
expect(result.cost).toEqual({
384+
input: 0.01,
385+
output: 0.02,
386+
total: expect.closeTo(0.032, 10),
387+
routing: 0.002,
388+
})
389+
})
390+
391+
it('leaves the reported model alone for an explicitly selected model', async () => {
392+
const result = (await handler.execute(mockContext, mockBlock, {
393+
model: 'gpt-4o',
394+
userPrompt: 'Hello!',
395+
apiKey: 'test-api-key',
396+
})) as { model: string }
397+
398+
expect(result.model).toBe('mock-model')
399+
})
400+
278401
it('should attach files to the last user message only', async () => {
279402
const inputs = {
280403
model: 'gpt-4o',

0 commit comments

Comments
 (0)