Skip to content

Commit 3a64923

Browse files
drewstonedrewstone
authored
feat(ci): toolchain volume build pipeline (#31)
* feat: pre-warm npm cache with Next.js + Tailwind stack for coinbase image Add next, react, react-dom, typescript, tailwindcss, postcss, autoprefixer, and @types/* packages to the coinbase project's cache_warm configuration. This populates the npm cache during image build so agent npm install hits cache instead of downloading ~200 transitive dependencies cold. Also fix npm cache permissions: the global npm install runs as root, creating root-owned cache entries. Added chown after install so the cache_warm step (running as agent) can write to the cache. Expected impact: reduce npm install from ~80-90s to ~10-20s in the coinbase-ecommerce benchmark scenario. * feat(ci): add toolchain volume build pipeline New CI workflow that builds toolchain layers as filesystem tarballs and publishes them as OCI artifacts to GHCR. This runs alongside the existing docker-publish.yml (which continues building images for backwards compatibility during migration). Workflow: 1. Build base toolchain from base-system.Dockerfile → export → push tarball 2. Build intermediate layers (foundry, rust, go, scipy) in parallel → tarballs 3. Generate + publish toolchain manifest (stack → layers mapping from config.json) Artifacts published to ghcr.io/tangle-network/toolchains/{layer}:{sha} with :latest tags. Hosts pull with oras and extract to /opt/toolchains/. See: agent-dev-container PR #459 and docs/rfcs/toolchain-volumes.md for the full architecture and deprecation plan.
1 parent ea4a31f commit 3a64923

File tree

4 files changed

+264
-4
lines changed

4 files changed

+264
-4
lines changed

‎.github/workflows/toolchain-volumes.yml‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
name: Build Toolchain Volumes
2+
3+
# Builds toolchain layers as filesystem tarballs and publishes them
4+
# as OCI artifacts to GHCR. These replace per-stack Docker images —
5+
# hosts download and extract them, then bind-mount onto thin base containers.
6+
#
7+
# See: agent-dev-container/docs/rfcs/toolchain-volumes.md
8+
9+
on:
10+
push:
11+
branches: [main]
12+
paths:
13+
- 'base/**'
14+
- 'intermediate/**'
15+
- 'config.json'
16+
- '.github/workflows/toolchain-volumes.yml'
17+
workflow_dispatch:
18+
inputs:
19+
layers:
20+
description: 'Layers to build (comma-separated, or "all")'
21+
default: 'all'
22+
type: string
23+
24+
concurrency:
25+
group: toolchain-volumes-${{ github.ref }}
26+
cancel-in-progress: true
27+
28+
env:
29+
REGISTRY: ghcr.io
30+
TOOLCHAIN_PREFIX: ghcr.io/${{ github.repository_owner }}/toolchains
31+
32+
jobs:
33+
# ─── Base toolchain: Ubuntu 24.04 + Node.js + Python + build tools + CLIs ───
34+
build-base:
35+
if: >
36+
github.event_name == 'push' ||
37+
contains(inputs.layers, 'all') ||
38+
contains(inputs.layers, 'base')
39+
runs-on: ubuntu-latest
40+
permissions:
41+
contents: read
42+
packages: write
43+
outputs:
44+
digest: ${{ steps.export.outputs.digest }}
45+
size: ${{ steps.export.outputs.size }}
46+
steps:
47+
- uses: actions/checkout@v4
48+
49+
- name: Set up Docker Buildx
50+
uses: docker/setup-buildx-action@v3
51+
52+
- name: Login to GHCR
53+
uses: docker/login-action@v3
54+
with:
55+
registry: ${{ env.REGISTRY }}
56+
username: ${{ github.actor }}
57+
password: ${{ secrets.GITHUB_TOKEN }}
58+
59+
- name: Build base toolchain image
60+
uses: docker/build-push-action@v6
61+
with:
62+
context: .
63+
file: base/base-system.Dockerfile
64+
platforms: linux/amd64
65+
load: true
66+
tags: toolchain-base:build
67+
cache-from: type=gha,scope=toolchain-base
68+
cache-to: type=gha,mode=max,scope=toolchain-base
69+
70+
- name: Export filesystem as tarball
71+
id: export
72+
run: |
73+
cid=$(docker create toolchain-base:build /bin/true)
74+
docker export "$cid" | gzip > /tmp/toolchain-base.tar.gz
75+
docker rm "$cid" > /dev/null
76+
77+
size=$(du -sh /tmp/toolchain-base.tar.gz | cut -f1)
78+
digest=$(sha256sum /tmp/toolchain-base.tar.gz | cut -d' ' -f1)
79+
echo "size=$size" >> "$GITHUB_OUTPUT"
80+
echo "digest=$digest" >> "$GITHUB_OUTPUT"
81+
echo "Base toolchain: $size (sha256:$digest)"
82+
83+
- name: Push as OCI artifact
84+
run: |
85+
# Push tarball to GHCR using oras
86+
curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | tar -xzf - -C /usr/local/bin oras
87+
oras push "${{ env.TOOLCHAIN_PREFIX }}/base:${{ github.sha }}" \
88+
--artifact-type "application/vnd.tangle.toolchain.v1+tar.gz" \
89+
/tmp/toolchain-base.tar.gz:application/gzip
90+
oras tag "${{ env.TOOLCHAIN_PREFIX }}/base:${{ github.sha }}" latest
91+
92+
- name: Make package public
93+
run: |
94+
gh api --method PUT /orgs/${{ github.repository_owner }}/packages/container/toolchains%2Fbase/visibility \
95+
-f visibility=public || true
96+
env:
97+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
98+
99+
# ─── Intermediate toolchains: foundry, rust, go, scientific-python ───
100+
build-intermediate:
101+
if: >
102+
github.event_name == 'push' ||
103+
contains(inputs.layers, 'all') ||
104+
contains(inputs.layers, 'foundry') ||
105+
contains(inputs.layers, 'rust') ||
106+
contains(inputs.layers, 'go') ||
107+
contains(inputs.layers, 'scientific-python')
108+
needs: build-base
109+
runs-on: ubuntu-latest
110+
permissions:
111+
contents: read
112+
packages: write
113+
strategy:
114+
fail-fast: false
115+
matrix:
116+
include:
117+
- name: foundry
118+
dockerfile: |
119+
FROM rust:latest
120+
ENV PATH=/root/.foundry/bin:/usr/local/cargo/bin:$PATH
121+
RUN curl -L https://foundry.paradigm.xyz | bash && \
122+
/root/.foundry/bin/foundryup && \
123+
chmod -R a+rx /root/.foundry
124+
- name: rust
125+
dockerfile: |
126+
FROM rust:latest
127+
RUN rustup target add wasm32-unknown-unknown 2>/dev/null || true
128+
- name: go
129+
dockerfile: |
130+
FROM golang:1.23-bookworm
131+
RUN go install golang.org/x/tools/gopls@latest 2>/dev/null || true
132+
- name: scientific-python
133+
dockerfile: |
134+
FROM python:3.12-slim
135+
RUN pip install --no-cache-dir numpy scipy pandas matplotlib scikit-learn jupyter 2>/dev/null || true
136+
steps:
137+
- uses: actions/checkout@v4
138+
139+
- name: Set up Docker Buildx
140+
uses: docker/setup-buildx-action@v3
141+
142+
- name: Login to GHCR
143+
uses: docker/login-action@v3
144+
with:
145+
registry: ${{ env.REGISTRY }}
146+
username: ${{ github.actor }}
147+
password: ${{ secrets.GITHUB_TOKEN }}
148+
149+
- name: Build ${{ matrix.name }} toolchain
150+
run: |
151+
echo '${{ matrix.dockerfile }}' > /tmp/Dockerfile.${{ matrix.name }}
152+
docker build -t toolchain-${{ matrix.name }}:build -f /tmp/Dockerfile.${{ matrix.name }} /tmp
153+
154+
- name: Export filesystem as tarball
155+
id: export
156+
run: |
157+
cid=$(docker create toolchain-${{ matrix.name }}:build /bin/true)
158+
docker export "$cid" | gzip > /tmp/toolchain-${{ matrix.name }}.tar.gz
159+
docker rm "$cid" > /dev/null
160+
161+
size=$(du -sh /tmp/toolchain-${{ matrix.name }}.tar.gz | cut -f1)
162+
digest=$(sha256sum /tmp/toolchain-${{ matrix.name }}.tar.gz | cut -d' ' -f1)
163+
echo "size=$size" >> "$GITHUB_OUTPUT"
164+
echo "digest=$digest" >> "$GITHUB_OUTPUT"
165+
echo "${{ matrix.name }} toolchain: $size (sha256:$digest)"
166+
167+
- name: Push as OCI artifact
168+
run: |
169+
curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | tar -xzf - -C /usr/local/bin oras
170+
oras push "${{ env.TOOLCHAIN_PREFIX }}/${{ matrix.name }}:${{ github.sha }}" \
171+
--artifact-type "application/vnd.tangle.toolchain.v1+tar.gz" \
172+
/tmp/toolchain-${{ matrix.name }}.tar.gz:application/gzip
173+
oras tag "${{ env.TOOLCHAIN_PREFIX }}/${{ matrix.name }}:${{ github.sha }}" latest
174+
175+
- name: Make package public
176+
run: |
177+
gh api --method PUT /orgs/${{ github.repository_owner }}/packages/container/toolchains%2F${{ matrix.name }}/visibility \
178+
-f visibility=public || true
179+
env:
180+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
181+
182+
# ─── Publish manifest mapping stacks to layers ───
183+
publish-manifest:
184+
needs: [build-base, build-intermediate]
185+
runs-on: ubuntu-latest
186+
permissions:
187+
contents: read
188+
packages: write
189+
steps:
190+
- uses: actions/checkout@v4
191+
192+
- name: Generate toolchain manifest
193+
run: |
194+
node -e "
195+
const config = require('./config.json');
196+
const projects = config.projects || {};
197+
const manifest = { version: '${{ github.sha }}', stacks: {} };
198+
199+
for (const [name, proj] of Object.entries(projects)) {
200+
if (typeof proj !== 'object') continue;
201+
const base = proj.base || 'base-system';
202+
const intermediate = base === 'base-system' ? null : base;
203+
manifest.stacks[name] = {
204+
intermediate: intermediate,
205+
packages: proj.packages || {},
206+
};
207+
}
208+
209+
require('fs').writeFileSync(
210+
'/tmp/toolchain-manifest.json',
211+
JSON.stringify(manifest, null, 2)
212+
);
213+
console.log('Generated manifest for', Object.keys(manifest.stacks).length, 'stacks');
214+
"
215+
216+
- name: Login to GHCR
217+
uses: docker/login-action@v3
218+
with:
219+
registry: ${{ env.REGISTRY }}
220+
username: ${{ github.actor }}
221+
password: ${{ secrets.GITHUB_TOKEN }}
222+
223+
- name: Push manifest as OCI artifact
224+
run: |
225+
curl -fsSL https://github.com/oras-project/oras/releases/download/v1.2.0/oras_1.2.0_linux_amd64.tar.gz | tar -xzf - -C /usr/local/bin oras
226+
oras push "${{ env.TOOLCHAIN_PREFIX }}/manifest:${{ github.sha }}" \
227+
--artifact-type "application/vnd.tangle.toolchain-manifest.v1+json" \
228+
/tmp/toolchain-manifest.json:application/json
229+
oras tag "${{ env.TOOLCHAIN_PREFIX }}/manifest:${{ github.sha }}" latest
230+
231+
- name: Make package public
232+
run: |
233+
gh api --method PUT /orgs/${{ github.repository_owner }}/packages/container/toolchains%2Fmanifest/visibility \
234+
-f visibility=public || true
235+
env:
236+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
237+
238+
- name: Summary
239+
run: |
240+
echo "## Toolchain Volumes Published" >> $GITHUB_STEP_SUMMARY
241+
echo "" >> $GITHUB_STEP_SUMMARY
242+
echo "| Layer | Artifact | Digest |" >> $GITHUB_STEP_SUMMARY
243+
echo "|-------|----------|--------|" >> $GITHUB_STEP_SUMMARY
244+
echo "| base | \`${{ env.TOOLCHAIN_PREFIX }}/base:${{ github.sha }}\` | \`${{ needs.build-base.outputs.digest }}\` |" >> $GITHUB_STEP_SUMMARY
245+
echo "| manifest | \`${{ env.TOOLCHAIN_PREFIX }}/manifest:${{ github.sha }}\` | — |" >> $GITHUB_STEP_SUMMARY
246+
echo "" >> $GITHUB_STEP_SUMMARY
247+
echo "Hosts pull with: \`oras pull ${{ env.TOOLCHAIN_PREFIX }}/base:latest\`" >> $GITHUB_STEP_SUMMARY

‎config.json‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,17 @@
2929
"@coinbase/agentkit@latest",
3030
"@x402/fetch@latest",
3131
"@x402/axios@latest",
32-
"@x402/express@latest"
32+
"@x402/express@latest",
33+
"next@latest",
34+
"react@latest",
35+
"react-dom@latest",
36+
"typescript@latest",
37+
"tailwindcss@latest",
38+
"postcss@latest",
39+
"autoprefixer@latest",
40+
"@types/react@latest",
41+
"@types/react-dom@latest",
42+
"@types/node@latest"
3343
]
3444
}
3545
},

‎generate_docker.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ function generateInfraDockerfile(project, config, outputDir) {
186186
if (packages.npm && packages.npm.length > 0) {
187187
const npmPackages = packages.npm.join(' ');
188188
dockerfileLines.push(`\nUSER root\n`);
189-
dockerfileLines.push(`RUN npm install -g ${npmPackages}\n`);
189+
dockerfileLines.push(`RUN npm install -g ${npmPackages} && chown -R agent:agent /tmp/.npm-cache 2>/dev/null || true\n`);
190190
dockerfileLines.push(`USER agent\n`);
191191
}
192192

@@ -325,7 +325,7 @@ function generateCombinedDockerfile(projectNames, outputDir) {
325325
if (uniqueNpmPackages.length > 0) {
326326
const npmPackages = uniqueNpmPackages.join(' ');
327327
dockerfileLines.push(`\nUSER root\n`);
328-
dockerfileLines.push(`RUN npm install -g ${npmPackages}\n`);
328+
dockerfileLines.push(`RUN npm install -g ${npmPackages} && chown -R agent:agent /tmp/.npm-cache 2>/dev/null || true\n`);
329329
dockerfileLines.push(`USER agent\n`);
330330
}
331331

‎infra/coinbase_ethereum.Dockerfile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
FROM foundry:latest
22

33
USER root
4-
RUN npm install -g @coinbase/coinbase-sdk ethers viem @wagmi/core
4+
RUN npm install -g @coinbase/coinbase-sdk @coinbase/onchainkit @coinbase/wallet-sdk @coinbase/cdp-sdk @coinbase/agentkit viem wagmi permissionless localtunnel ethers @wagmi/core hardhat @nomicfoundation/hardhat-toolbox && chown -R agent:agent /tmp/.npm-cache 2>/dev/null || true
55
USER agent
66

7+
# Pre-warm npm cache with project-specific packages
8+
RUN npm cache add @coinbase/onchainkit@latest @coinbase/wallet-sdk@latest @coinbase/cdp-sdk@latest @coinbase/agentkit@latest @x402/fetch@latest @x402/axios@latest @x402/express@latest next@latest react@latest react-dom@latest typescript@latest tailwindcss@latest postcss@latest autoprefixer@latest @types/react@latest @types/react-dom@latest @types/node@latest @openzeppelin/contracts@latest @openzeppelin/contracts-upgradeable@latest wagmi@latest @rainbow-me/rainbowkit@latest abitype@latest || true
9+
710
LABEL description="Combined: coinbase, ethereum"

0 commit comments

Comments
 (0)