Skip to content
Test and Build

Test and Build #200

Sign in to view logs

Test and Build

Test and Build #200

Workflow file for this run

.github/workflows/test-build.yml at 2a79c85
name: Test and Build
on:
workflow_call:
workflow_dispatch:
inputs:
comparison_base:
description: 'Full commit SHA to compare against the selected branch'
required: true
type: string
permissions:
contents: read
jobs:
oauth-postgres:
name: PostgreSQL integration (${{ matrix.provision }})
runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
provision: [push, migrate]
services:
postgres:
image: pgvector/pgvector:pg17
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: sim_auth_scim
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres -d sim_auth_scim"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
OAUTH_TOKEN_FAMILY_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
BETTER_AUTH_SECRET: oauth-postgres-ci-secret-at-least-32-characters
NEXT_PUBLIC_APP_URL: https://test.sim.ai
ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000'
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.1
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 24
- name: Mount Bun cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ~/.bun/install/cache
- name: Install dependencies
run: bun install --frozen-lockfile --ignore-scripts
- name: Provision a fresh database through the supported command
working-directory: packages/db
run: |
bun -e 'import postgres from "postgres"; const sql = postgres(process.env.DATABASE_URL); for (const extension of ["vector", "btree_gin", "pg_trgm"]) await sql`CREATE EXTENSION IF NOT EXISTS ${sql(extension)}`; await sql.end()'
bun run db:${{ matrix.provision }}
- name: Verify migration replay is a no-op
if: matrix.provision == 'migrate'
working-directory: packages/db
run: bun run db:migrate
- name: Verify OAuth lifecycle and SCIM membership guards in PostgreSQL
working-directory: apps/sim
run: >-
bunx vitest run
lib/auth/oauth-token-family.postgres.test.ts
lib/auth/oauth-provider-lifecycle.postgres.test.ts
app/api/auth/oauth2/token/route.postgres.test.ts
lib/auth/sim-auth-adapter.test.ts
ee/scim/lib/managed-membership.postgres.test.ts
lib/auth/sso/application/admit-sso-user.postgres.test.ts
- name: Verify SCIM and administration over real HTTP
working-directory: apps/sim
env:
NEXT_PUBLIC_APP_URL: http://127.0.0.1:3017
BETTER_AUTH_URL: http://127.0.0.1:3017
NEXT_PUBLIC_FORCE_HOSTED: 'true'
BILLING_ENABLED: 'true'
NEXT_PUBLIC_BILLING_ENABLED: 'true'
ENTERPRISE_ENABLED: 'true'
NEXT_PUBLIC_ENTERPRISE_ENABLED: 'true'
SCIM_ENABLED: 'true'
NEXT_PUBLIC_SCIM_ENABLED: 'true'
SSO_ENABLED: 'true'
NEXT_PUBLIC_SSO_ENABLED: 'true'
ORGANIZATIONS_ENABLED: 'true'
NEXT_PUBLIC_ORGANIZATIONS_ENABLED: 'true'
INTERNAL_API_SECRET: scim-http-ci-local-secret-at-least-32-characters
DB_TX_TRIPWIRE: throw
DISABLE_TELEMETRY: 'true'
NEXT_TELEMETRY_DISABLED: '1'
NEXT_PUBLIC_CHAT_DISABLED: 'true'
run: |
server_log="$RUNNER_TEMP/scim-next.log"
node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 --port 3017 > "$server_log" 2>&1 &
server_pid=$!
finish() {
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$RUNNER_TEMP/scim-http-status.log"
}
trap finish EXIT
deadline=$((SECONDS + 120))
until curl --fail --silent --max-time 3 http://127.0.0.1:3017/api/health > /dev/null; do
if ! kill -0 "$server_pid" 2>/dev/null; then
echo 'Local SCIM app exited during startup.'
exit 1
fi
if [ "$SECONDS" -ge "$deadline" ]; then
echo 'Local SCIM app did not become ready within 120 seconds.'
exit 1
fi
sleep 2
done
SCIM_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \
SCIM_E2E_DATABASE_URL="$DATABASE_URL" \
SCIM_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \
SCIM_E2E_REPORT_PATH="$RUNNER_TEMP/scim-e2e-report.json" \
bun run test:scim:e2e
- name: Upload SCIM failure report and HTTP status log
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: scim-failure-${{ matrix.provision }}
path: |
${{ runner.temp }}/scim-e2e-report.json
${{ runner.temp }}/scim-http-status.log
if-no-files-found: ignore
retention-days: 7
- name: Verify durable provenance bindings and concurrent memory writes
working-directory: apps/sim
env:
TABLE_PROVENANCE_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
MEMORY_PROVENANCE_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
run: >-
bunx vitest run
lib/table/rows/secret-provenance.postgres.test.ts
lib/memory/message-provenance.postgres.test.ts
- name: Verify Search progress and pagination in PostgreSQL
working-directory: apps/sim
env:
KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
run: >-
bunx vitest run --mode integration
lib/knowledge/__integration__/search-source-progress.integration.ts
lib/knowledge/__integration__/search-source-pagination.integration.ts
test-build:
name: Lint and Test
runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
timeout-minutes: 15
steps:
# The diff-based audits below need a base commit to read, and the default
# depth of 1 clones a single commit with no parent. They normally fetch
# their base by SHA (see "Resolve base ref"), so this depth only covers the
# new-branch push's `HEAD~1` fallback — without it that resolves to nothing.
#
# Worth stating because the failure was invisible for so long: the migration
# audit read the resulting `git diff` failure as "no migrations changed" and
# exited 0, so it had never actually run on a push build.
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
fetch-depth: 2
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.1
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 24
# Cache keys are scoped by event name, and fork PRs get their own
# namespace on top: untrusted fork runs must never share a cache with
# push runs (whose caches feed production image builds) or with trusted
# internal-PR runs.
#
# node_modules also keys on the lockfile hash: a sticky disk is a mutable
# volume, and `bun install --frozen-lockfile` adds what the lockfile needs
# without pruning what it dropped, so branches on different lockfiles were
# contaminating each other (a stale @next/swc 16.2.6 outlived the 16.2.11
# bump). The bun and Turbo caches are content/hash-addressed, so they stay
# shared — that is what keeps a fresh node_modules disk cheap to fill.
- name: Mount Bun cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ~/.bun/install/cache
- name: Mount node_modules
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }}
path: ./node_modules
- name: Mount Turbo cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-turbo-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ./.turbo
- name: Install dependencies
run: bun install --frozen-lockfile --ignore-scripts
# Surfaces known CVEs in the dependency tree. Non-blocking until the
# existing advisory backlog is triaged, then flip to a required gate by
# removing continue-on-error.
- name: Security audit
run: bun audit
continue-on-error: true
- name: Validate env flags
run: |
FILE="apps/sim/lib/core/config/env-flags.ts"
ERRORS=""
echo "Checking for hardcoded boolean env flags..."
# Use perl for multiline matching to catch both:
# export const isHosted = true
# export const isHosted =
# true
HARDCODED=$(perl -0777 -ne 'while (/export const (is[A-Za-z]+)\s*=\s*\n?\s*(true|false)\b/g) { print " $1 = $2\n" }' "$FILE")
if [ -n "$HARDCODED" ]; then
ERRORS="${ERRORS}\n❌ Env flags must not be hardcoded to boolean literals!\n\nFound hardcoded flags:\n${HARDCODED}\n\nEnv flags should derive their values from environment variables.\n"
fi
echo "Checking env flag naming conventions..."
# Check that all export const (except functions) start with 'is'
# This finds exports like "export const someFlag" that don't start with "is" or "get"
BAD_NAMES=$(grep -E "^export const [a-z]" "$FILE" | grep -vE "^export const (is|get)" | sed 's/export const \([a-zA-Z]*\).*/ \1/')
if [ -n "$BAD_NAMES" ]; then
ERRORS="${ERRORS}\n❌ Env flags must use 'is' prefix for boolean flags!\n\nFound incorrectly named flags:\n${BAD_NAMES}\n\nExample: 'hostedMode' should be 'isHostedMode'\n"
fi
if [ -n "$ERRORS" ]; then
echo ""
echo -e "$ERRORS"
exit 1
fi
echo "✅ All env flags are properly configured"
# One fetch for both base-ref audits, and no `|| true`: a swallowed fetch leaves
# the base ref absent, which neither audit can tell apart from a branch that
# changed nothing. The block-registry check at least degrades to a visible
# `⚠ … skipping` line; the migration audit printed `✓ No new migrations to
# check` and exited 0, clearing the only guard on production DDL.
#
# Depth stays at 1 — without a merge-base the migration audit diffs the two
# tips, which under `--diff-filter=AM` is exactly the migrations new here.
# Resolved once for both diff-based audits, and never with `|| true`: a
# swallowed fetch leaves the base absent, which neither audit can tell apart
# from a branch that changed nothing.
#
# On push the base is `github.event.before`, the tip the branch had before
# this push — not `HEAD~1`, which names only the last commit and would let a
# multi-commit push slip every earlier commit's migrations past the audit.
# It is fetched by SHA at depth 1; the audits diff two tips and need no
# common ancestry. An all-zero `before` means the branch is new and has no
# predecessor to diff, so `HEAD~1` remains the fallback there.
# PRs use the event's base SHA, not a branch that can advance while queued.
# Manual runs require an explicit base so earlier commits are audited too.
- name: Resolve base ref for diff-based audits
id: audit_base
env:
EVENT_NAME: ${{ github.event_name }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
PUSH_BEFORE_SHA: ${{ github.event.before }}
COMPARISON_BASE: ${{ inputs.comparison_base }}
run: |
if [ "$EVENT_NAME" = "pull_request" ]; then
base_sha="$PR_BASE_SHA"
elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then
base_sha="$COMPARISON_BASE"
elif [ -n "$PUSH_BEFORE_SHA" ] &&
[ "$PUSH_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]; then
base_sha="$PUSH_BEFORE_SHA"
else
base_sha="$(git rev-parse --verify 'HEAD~1^{commit}')"
fi
if ! [[ "$base_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then
echo 'Comparison base must be a full commit SHA.' >&2
exit 1
fi
git fetch --no-tags --depth=1 origin "$base_sha"
resolved_base="$(git rev-parse --verify "$base_sha^{commit}")"
echo "ref=$resolved_base" >> "$GITHUB_OUTPUT"
- name: Check block registry invariants
run: bun run apps/sim/scripts/check-block-registry.ts "${{ steps.audit_base.outputs.ref }}"
- name: Lint code
run: bun run lint:check
# Every zero-argument `check:*` script, run concurrently. The list is derived in
# scripts/run-audits.ts, which also writes the per-audit timing table to the job
# summary and annotates failures. Audits needing a base ref stay separate below.
- name: Repo audits
run: bun run check:audits
- name: Verify docs manifest is in sync
run: bun run docs-manifest:check
- name: Migration safety (zero-downtime) audit
run: bun run check:migrations "${{ steps.audit_base.outputs.ref }}"
# Every workspace, not just realtime. packages/emcn, packages/utils,
# apps/desktop and apps/docs had no type check in CI at all; apps/sim's
# source was covered only as a side effect of `next build` in the separate
# Build App job. Note this does NOT cover apps/sim's tests — its tsconfig
# excludes *.test.ts(x), and including them today surfaces ~2.2k errors,
# so that is its own cleanup rather than a gate to switch on here.
- name: Type-check all workspaces
run: bunx turbo run type-check
# cloud-review-tools.test.ts runs the real helper on the runner, which shells
# out to rg. Blacksmith's image ships it, GitHub's doesn't.
- name: Install ripgrep
run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep)
# Runs the setup CLI's Bun tests plus each workspace's Vitest suite,
# without `--coverage`. See the Codecov note below.
#
# apps/sim runs only its first shard here; `test-shard` below runs the
# others. That suite is bound by the single Vite server thread that feeds
# every worker — wall time is flat from 4 to 13 workers — so a bigger
# runner buys nothing and each extra runner takes a proportional slice.
- name: Run tests
env:
NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
NEXT_PUBLIC_APP_URL: 'https://www.sim.ai'
DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only
TURBO_CACHE_DIR: .turbo
SIM_TEST_SHARD: 1/3
run: bun run test
- name: Check schema and migrations are in sync
working-directory: packages/db
run: |
bunx drizzle-kit generate --config=./drizzle.config.ts
if [ -n "$(git status --porcelain ./migrations)" ]; then
echo "❌ Schema and migrations are out of sync!"
echo "Run 'cd packages/db && bunx drizzle-kit generate' and commit the new migrations."
git status --porcelain ./migrations
git diff ./migrations
exit 1
fi
echo "✅ Schema and migrations are in sync"
# The remaining shards of apps/sim's Vitest suite. Everything else — lint,
# the audits, type-check, the other workspaces' suites — lives in
# `test-build` with shard 1; these jobs exist only because that suite cannot
# go faster on one machine (see the "Run tests" note there). Three shards
# put each runner at roughly the fixed cost of checkout + install. The Turbo
# cache disk gets its own key so the shards' entries do not evict each other.
test-shard:
name: Test (shard ${{ matrix.shard }})
runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
shard: [2, 3]
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.1
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 24
- name: Mount Bun cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ~/.bun/install/cache
- name: Mount node_modules
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }}
path: ./node_modules
- name: Mount Turbo cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-turbo-cache-shard-${{ matrix.shard }}-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ./.turbo
- name: Install dependencies
run: bun install --frozen-lockfile --ignore-scripts
- name: Install ripgrep
run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep)
- name: Run tests (apps/sim shard ${{ matrix.shard }}/3)
env:
NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
NEXT_PUBLIC_APP_URL: 'https://www.sim.ai'
DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only
TURBO_CACHE_DIR: .turbo
SIM_TEST_SHARD: ${{ matrix.shard }}/3
run: bunx turbo run test --filter=@sim/app
# Next.js production build, in parallel with lint + tests. Sticky disks are
# cloned from the last committed snapshot per job and committed last-writer-
# wins, so concurrent mounts are safe. The bun/node_modules disks are shared
# with test-build (the lockfile-hashed key means they only ever share when the
# dependency tree really is identical, so LWW loss is harmless), but the Turbo
# cache gets its own key: with a shared key, only the last committer's new
# entries survive each run, so the test and build Turbo entries would evict
# each other nondeterministically.
#
# Runner is sized for the COLD-cache build, which is what OOM-killed the 8vcpu
# tier (23 kills / 1074 runs at 98% of its 30.4 GB): warm peaks ~12 GB, cold
# peaked 51 GB. NODE_OPTIONS' --max-old-space-size caps only Node's JS heap,
# not the native Turbopack workers that dominate, so it cannot prevent this.
build:
name: Build App
runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }}
# Build durations crossed 15 minutes as the app grew (10m02 on Jul 29 AM,
# 14m44 after the folders/desktop/library merges, then two straight
# timeouts) — GitHub reports a job timeout as "cancelled". 25 keeps
# headroom without masking a genuine hang.
timeout-minutes: 25
steps:
- name: Checkout code
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.1
- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 24
- name: Mount Bun cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ~/.bun/install/cache
- name: Mount node_modules
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }}
path: ./node_modules
- name: Mount Turbo cache
uses: ./.github/actions/cache-mount
with:
provider: ${{ vars.CI_PROVIDER }}
key: ${{ github.repository }}-turbo-cache-build-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
path: ./.turbo
# No `.next/cache` mount: the Turbopack persistent build cache is off. A
# controlled A/B on one branch (PR #6078) with a byte-identical module graph
# measured compile at 113s with the cache off, 162s cold with it on, and
# 360s warm — the cache made the same build 3.2x slower, and it grew
# 5.1 GB -> 12 GB across two runs of an unchanged tree, so a disk degrades
# the more it is used. Mounting a disk nothing reads would only cost storage.
# Running out of RAM kills the whole VM and surfaces only as "the runner
# has received a shutdown signal" — no mention of memory, ~12 min in. Warn
# with the real numbers so that failure is a one-line diagnosis instead of
# a mystery. Warn, never fail: a warm build peaks ~12 GB and a partial one
# ~28 GB, so a 32 GB runner still completes plenty of builds, and the
# GitHub fallback is the break-glass path — degrading it to a guaranteed
# failure would be worse than the risk this flags.
- name: Check runner memory headroom
run: |
TOTAL_GB=$(awk '/MemTotal/ {printf "%d", $2/1048576}' /proc/meminfo)
echo "Runner memory: ${TOTAL_GB} GB"
if [ "$TOTAL_GB" -lt 40 ]; then
echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit."
fi
- name: Install dependencies
run: bun install --frozen-lockfile --ignore-scripts
- name: Build application
env:
NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
NEXT_PUBLIC_APP_URL: 'https://www.sim.ai'
DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
STRIPE_SECRET_KEY: 'dummy_key_for_ci_only'
STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only'
RESEND_API_KEY: 'dummy_key_for_ci_only'
AWS_REGION: 'us-west-2'
ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only
TURBO_CACHE_DIR: .turbo
run: bunx turbo run build --filter=@sim/app