diff --git a/.github/workflows/llm_chatbot.yml b/.github/workflows/llm_chatbot.yml new file mode 100644 index 0000000000..59943698b4 --- /dev/null +++ b/.github/workflows/llm_chatbot.yml @@ -0,0 +1,28 @@ +name: llm_chatbot + +on: + push: + branches: [master] + pull_request: + paths: + - rust/llm_chatbot/** + - .github/workflows/llm_chatbot.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust-llm_chatbot: + runs-on: ubuntu-24.04 + container: ghcr.io/dfinity/icp-dev-env-rust:1.0.1 + env: + ICP_CLI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Deploy and test + working-directory: rust/llm_chatbot + run: | + icp network start -d + icp deploy + bash test.sh diff --git a/rust/llm_chatbot/.devcontainer/devcontainer.json b/rust/llm_chatbot/.devcontainer/devcontainer.json deleted file mode 100644 index ebb0b8bcc6..0000000000 --- a/rust/llm_chatbot/.devcontainer/devcontainer.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "ICP Dev Environment", - "image": "ghcr.io/dfinity/icp-dev-env-slim:22", - "forwardPorts": [4943, 5173], - "portsAttributes": { - "4943": { - "label": "dfx", - "onAutoForward": "ignore" - }, - "5173": { - "label": "vite", - "onAutoForward": "openBrowser" - } - }, - "customizations": { - "vscode": { - "extensions": ["dfinity-foundation.vscode-motoko"] - } - } -} diff --git a/rust/llm_chatbot/BUILD.md b/rust/llm_chatbot/BUILD.md deleted file mode 100644 index 24cfcb7547..0000000000 --- a/rust/llm_chatbot/BUILD.md +++ /dev/null @@ -1,113 +0,0 @@ -# Continue building locally - -Projects deployed through ICP Ninja are temporary; they will only be live for 20 minutes before they are removed. The command-line tool `dfx` can be used to continue building your ICP Ninja project locally and deploy it to the mainnet. - -To migrate your ICP Ninja project off of the web browser and develop it locally, follow these steps. - -### 1. Install developer tools. - -You can install the developer tools natively or use Dev Containers. - -#### Option 1: Natively install developer tools - -> Installing `dfx` natively is currently only supported on macOS and Linux systems. On Windows, it is recommended to use the Dev Containers option. - -1. Install `dfx` with the following command: - -``` - -sh -ci "$(curl -fsSL https://internetcomputer.org/install.sh)" - -``` - -> On Apple Silicon (e.g., Apple M1 chip), make sure you have Rosetta installed (`softwareupdate --install-rosetta`). - -2. [Install NodeJS](https://nodejs.org/en/download/package-manager). - -3. For Rust projects, you will also need to: - -- Install [Rust](https://doc.rust-lang.org/cargo/getting-started/installation.html#install-rust-and-cargo): `curl https://sh.rustup.rs -sSf | sh` - -- Install [candid-extractor](https://crates.io/crates/candid-extractor): `cargo install candid-extractor` - -4. For Motoko projects, you will also need to: - -- Install the Motoko package manager [Mops](https://docs.mops.one/quick-start#2-install-mops-cli): `npm i -g ic-mops` - -Lastly, navigate into your project's directory that you downloaded from ICP Ninja. - -#### Option 2: Dev Containers - -Continue building your projects locally by installing the [Dev Container extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) for VS Code and [Docker](https://docs.docker.com/engine/install/). - -Make sure Docker is running, then navigate into your project's directory that you downloaded from ICP Ninja and start the Dev Container by selecting `Dev-Containers: Reopen in Container` in VS Code's command palette (F1 or Ctrl/Cmd+Shift+P). - -> Note that local development ports (e.g. the ports used by `dfx` or `vite`) are forwarded from the Dev Container to your local machine. In the VS code terminal, use Ctrl/Cmd+Click on the displayed local URLs to open them in your browser. To view the current port mappings, click the "Ports" tab in the VS Code terminal window. - -### 2. Start the local development environment. - -``` -dfx start --background -``` - -### 3. Create a local developer identity. - -To manage your project's canisters, it is recommended that you create a local [developer identity](https://internetcomputer.org/docs/building-apps/getting-started/identities) rather than use the `dfx` default identity that is not stored securely. - -To create a new identity, run the commands: - -``` - -dfx identity new IDENTITY_NAME - -dfx identity use IDENTITY_NAME - -``` - -Replace `IDENTITY_NAME` with your preferred identity name. The first command `dfx start --background` starts the local `dfx` processes, then `dfx identity new` will create a new identity and return your identity's seed phase. Be sure to save this in a safe, secure location. - -The third command `dfx identity use` will tell `dfx` to use your new identity as the active identity. Any canister smart contracts created after running `dfx identity use` will be owned and controlled by the active identity. - -Your identity will have a principal ID associated with it. Principal IDs are used to identify different entities on ICP, such as users and canisters. - -[Learn more about ICP developer identities](https://internetcomputer.org/docs/building-apps/getting-started/identities). - -### 4. Deploy the project locally. - -Deploy your project to your local developer environment with: - -``` -npm install -dfx deploy - -``` - -Your project will be hosted on your local machine. The local canister URLs for your project will be shown in the terminal window as output of the `dfx deploy` command. You can open these URLs in your web browser to view the local instance of your project. - -### 5. Obtain cycles. - -To deploy your project to the mainnet for long-term public accessibility, first you will need [cycles](https://internetcomputer.org/docs/building-apps/getting-started/tokens-and-cycles). Cycles are used to pay for the resources your project uses on the mainnet, such as storage and compute. - -> This cost model is known as ICP's [reverse gas model](https://internetcomputer.org/docs/building-apps/essentials/gas-cost), where developers pay for their project's gas fees rather than users pay for their own gas fees. This model provides an enhanced end user experience since they do not need to hold tokens or sign transactions when using a dapp deployed on ICP. - -> Learn how much a project may cost by using the [pricing calculator](https://internetcomputer.org/docs/building-apps/essentials/cost-estimations-and-examples). - -Cycles can be obtained through [converting ICP tokens into cycles using `dfx`](https://internetcomputer.org/docs/building-apps/developer-tools/dfx/dfx-cycles#dfx-cycles-convert). - -### 6. Deploy to the mainnet. - -Once you have cycles, run the command: - -``` - -dfx deploy --network ic - -``` - -After your project has been deployed to the mainnet, it will continuously require cycles to pay for the resources it uses. You will need to [top up](https://internetcomputer.org/docs/building-apps/canister-management/topping-up) your project's canisters or set up automatic cycles management through a service such as [CycleOps](https://cycleops.dev/). - -> If your project's canisters run out of cycles, they will be removed from the network. - -## Additional examples - -Additional code examples and sample applications can be found in the [DFINITY examples repo](https://github.com/dfinity/examples). diff --git a/rust/llm_chatbot/README.md b/rust/llm_chatbot/README.md index 409b4e9b65..c011f17787 100644 --- a/rust/llm_chatbot/README.md +++ b/rust/llm_chatbot/README.md @@ -1,50 +1,73 @@ # LLM Chatbot -The LLM Chatbot example demonstrates how an ICP smart contract can be used to interact with a large language model (LLM) to generate text. The user can input a prompt, and the smart contract will use the LLM to generate a response. -The response is then returned to the user, and the user can submit some follow-up prompts to continue the conversation. +This example demonstrates how an ICP smart contract can interact with a large language model (LLM) to generate text. The user can input a prompt and the smart contract will use the LLM to generate a response. Follow-up prompts continue the conversation with the full message history. -## Deploying from ICP Ninja +## How it works -When viewing this project in ICP Ninja, you can deploy it directly to the mainnet for free by clicking "Run" in the upper right corner. Open this project in ICP Ninja: +The backend canister calls the `w36hm-eqaaa-aaaal-qr76a-cai` LLM canister on mainnet via the [`ic-llm`](https://crates.io/crates/ic-llm) crate. Locally, `icp deploy` deploys a copy of the LLM canister (backed by Ollama) and automatically injects its canister ID into the backend as the `PUBLIC_CANISTER_ID:llm` environment variable — `ic-llm` reads this at runtime so local calls go to the local LLM. On mainnet, the env var is absent and `ic-llm` falls back to the hardcoded mainnet principal. -[![](https://icp.ninja/assets/open.svg)](https://icp.ninja/i?g=https://github.com/dfinity/examples/rust/llm_chatbot) +## Build and deploy from the command line -## Deploying from ICP Ninja +### Prerequisites -[![](https://icp.ninja/assets/open.svg)](https://icp.ninja/editor?g=https://github.com/dfinity/examples/tree/master/rust/counter) +- Node.js +- icp-cli: `npm install -g @icp-sdk/icp-cli @icp-sdk/ic-wasm` +- ic-mops: `npm install -g ic-mops` -## Build and deploy from the command-line +### Set up Ollama (local deployment only) -### 1. [Download and install the IC SDK.](https://internetcomputer.org/docs/building-apps/getting-started/install) +The LLM canister delegates inference to [Ollama](https://ollama.com/). Install it and then run: -### 2. Download your project from ICP Ninja using the 'Download files' button on the upper left corner, or [clone the GitHub examples repository.](https://github.com/dfinity/examples/) +```bash +ollama serve +``` + +In a separate terminal, pull the model (about 4 GiB, one-time download): -### 3. Navigate into the project's directory. +```bash +ollama pull llama3.1:8b +``` -### 4. Set up Ollama +Once the model is loaded you can stop the `ollama run` command — `ollama serve` keeps it available. -To be able to test the agent locally, you'll need a server for processing the agent's prompts. For that, we'll use `ollama`, which is a tool that can download and serve LLMs. -See the documentation on the [Ollama website](https://ollama.com/) to install it. Once it's installed, run: +### Install +```bash +git clone https://github.com/dfinity/examples +cd examples/rust/llm_chatbot ``` -ollama serve -# Expected to start listening on port 11434 + +### Deploy and test + +```bash +icp network start -d +icp deploy +bash test.sh +icp network stop ``` -The above command will start the Ollama server, so that it can process requests by the agent. Additionally, and in a separate window, run the following command to download the LLM that will be used by the agent: +Open the frontend URL printed by `icp deploy` to use the chatbot in the browser. + +For hot-reload frontend development: +```bash +npm run dev --prefix frontend ``` -ollama run llama3.1:8b + +## Deploying to mainnet + +```bash +icp deploy -e ic ``` -The above command will download an 8B parameter model, which is around 4GiB. Once the command executes and the model is loaded, you can terminate it. You won't need to do this step again. +No Ollama setup is needed — mainnet calls go directly to the LLM canister at `w36hm-eqaaa-aaaal-qr76a-cai`. -### 5. Deploy the project to your local environment: +## Updating the Candid interface -``` -dfx start --background --clean && dfx deploy +```bash +icp build backend && candid-extractor target/wasm32-unknown-unknown/release/backend.wasm > backend/backend.did ``` ## Security considerations and best practices -If you base your application on this example, it is recommended that you familiarize yourself with and adhere to the [security best practices](https://internetcomputer.org/docs/building-apps/security/overview) for developing on ICP. This example may not implement all the best practices. +If you base your application on this example, familiarize yourself with the [security best practices](https://docs.internetcomputer.org/guides/security/overview) for developing on ICP. This example may not implement all best practices. diff --git a/rust/llm_chatbot/backend/Cargo.toml b/rust/llm_chatbot/backend/Cargo.toml index acb70dd01d..aaf0d3e933 100644 --- a/rust/llm_chatbot/backend/Cargo.toml +++ b/rust/llm_chatbot/backend/Cargo.toml @@ -1,15 +1,13 @@ [package] name = "backend" version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +edition = "2024" [lib] crate-type = ["cdylib"] path = "lib.rs" [dependencies] -candid = "0.10.13" -ic-cdk = "0.17.1" -ic-llm = "1.1.0" +candid = "0.10" +ic-cdk = "0.20" +ic-llm = "1.2.0" diff --git a/rust/llm_chatbot/backend/backend.did b/rust/llm_chatbot/backend/backend.did new file mode 100644 index 0000000000..0ae3e1c09c --- /dev/null +++ b/rust/llm_chatbot/backend/backend.did @@ -0,0 +1,17 @@ +type AssistantMessage = record { + content : opt text; + tool_calls : vec ToolCall; +}; +type ChatMessage = variant { + tool : record { content : text; tool_call_id : text }; + user : record { content : text }; + assistant : AssistantMessage; + system : record { content : text }; +}; +type FunctionCall = record { name : text; arguments : vec ToolCallArgument }; +type ToolCall = record { id : text; function : FunctionCall }; +type ToolCallArgument = record { value : text; name : text }; +service : { + chat : (vec ChatMessage) -> (text); + prompt : (text) -> (text); +} diff --git a/rust/llm_chatbot/dfx.json b/rust/llm_chatbot/dfx.json deleted file mode 100644 index 778ae95ae1..0000000000 --- a/rust/llm_chatbot/dfx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "canisters": { - "backend": { - "candid": "backend/backend.did", - "type": "custom", - "shrink": true, - "gzip": true, - "wasm": "target/wasm32-unknown-unknown/release/backend.wasm", - "build": [ - "cargo build --target wasm32-unknown-unknown --release -p backend", - "candid-extractor target/wasm32-unknown-unknown/release/backend.wasm > backend/backend.did" - ], - "metadata": [ - { - "name": "candid:service" - } - ] - }, - "frontend": { - "dependencies": ["backend"], - "frontend": { - "entrypoint": "frontend/index.html" - }, - "source": ["frontend/dist"], - "type": "assets" - }, - "llm": { - "candid": "https://github.com/dfinity/llm/releases/latest/download/llm-canister-ollama.did", - "type": "custom", - "specified_id": "w36hm-eqaaa-aaaal-qr76a-cai", - "remote": { - "id": { - "ic": "w36hm-eqaaa-aaaal-qr76a-cai" - } - }, - "wasm": "https://github.com/dfinity/llm/releases/latest/download/llm-canister-ollama.wasm" - } - }, - "output_env_file": ".env" -} diff --git a/rust/llm_chatbot/frontend/package.json b/rust/llm_chatbot/frontend/package.json index fd97b3e4b4..8e0f45f974 100644 --- a/rust/llm_chatbot/frontend/package.json +++ b/rust/llm_chatbot/frontend/package.json @@ -3,24 +3,21 @@ "private": true, "type": "module", "scripts": { - "prebuild": "npm i --include=dev && dfx generate backend", + "prebuild": "npm i --include=dev", "build": "vite build", "dev": "vite" }, "dependencies": { "@icp-sdk/core": "~5.2.0", - "react-json-view-lite": "2.3.0", "react": "18.3.1", "react-dom": "18.3.1" }, "devDependencies": { - "@types/react": "18.3.12", - "@types/react-dom": "18.3.1", + "@icp-sdk/bindgen": "~0.2.2", "@vitejs/plugin-react": "4.3.3", "autoprefixer": "^10.4.20", "postcss": "8.4.48", "tailwindcss": "3.4.14", - "vite": "5.4.11", - "vite-plugin-environment": "1.1.3" + "vite": "5.4.11" } } diff --git a/rust/llm_chatbot/frontend/src/actor.js b/rust/llm_chatbot/frontend/src/actor.js new file mode 100644 index 0000000000..4b4bc18cd5 --- /dev/null +++ b/rust/llm_chatbot/frontend/src/actor.js @@ -0,0 +1,22 @@ +import { safeGetCanisterEnv } from "@icp-sdk/core/agent/canister-env"; +import { createActor } from "./bindings/backend"; + +// The ic_env cookie is set by the asset canister on all HTML responses. It +// contains the replica root key and PUBLIC_* canister environment variables. +// In dev mode the Vite dev server sets the same cookie (see vite.config.js). +const canisterEnv = safeGetCanisterEnv(); + +const canisterId = canisterEnv?.["PUBLIC_CANISTER_ID:backend"]; + +if (!canisterId) { + throw new Error( + "Canister ID for 'backend' not found. Run 'icp deploy' first." + ); +} + +export const backend = createActor(canisterId, { + agentOptions: { + host: window.location.origin, + rootKey: canisterEnv?.IC_ROOT_KEY, + }, +}); diff --git a/rust/llm_chatbot/frontend/src/main.jsx b/rust/llm_chatbot/frontend/src/main.jsx index 9cf706bf64..ce01631e27 100644 --- a/rust/llm_chatbot/frontend/src/main.jsx +++ b/rust/llm_chatbot/frontend/src/main.jsx @@ -1,6 +1,6 @@ import React, { useState, useRef, useEffect } from 'react'; import ReactDOM from 'react-dom/client'; -import { backend } from 'declarations/backend'; +import { backend } from './actor'; import botImg from '/bot.svg'; import userImg from '/user.svg'; import '/index.css'; diff --git a/rust/llm_chatbot/frontend/vite.config.js b/rust/llm_chatbot/frontend/vite.config.js index f9e04a9a93..be7ec53902 100644 --- a/rust/llm_chatbot/frontend/vite.config.js +++ b/rust/llm_chatbot/frontend/vite.config.js @@ -1,37 +1,67 @@ -import react from '@vitejs/plugin-react'; -import { defineConfig } from 'vite'; -import { fileURLToPath, URL } from 'url'; -import environment from 'vite-plugin-environment'; - -export default defineConfig({ - base: './', - plugins: [react(), environment('all', { prefix: 'CANISTER_' }), environment('all', { prefix: 'DFX_' })], - envDir: '../', - define: { - 'process.env': process.env - }, - optimizeDeps: { - esbuildOptions: { - define: { - global: 'globalThis' - } - } - }, - resolve: { - alias: [ - { - find: 'declarations', - replacement: fileURLToPath(new URL('../src/declarations', import.meta.url)) - } - ] - }, - server: { +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; +import { icpBindgen } from "@icp-sdk/bindgen/plugins/vite"; +import { execSync } from "child_process"; + +export default defineConfig(({ command }) => { + const plugins = [ + react(), + icpBindgen({ + didFile: "../backend/backend.did", + outDir: "./src/bindings", + }), + ]; + + // Build only — no dev-server setup needed + if (command !== "serve") { + return { plugins }; + } + + // Dev server: look up the local network root key and backend canister ID + const environment = process.env.ICP_ENVIRONMENT || "local"; + const CANISTER_NAME = "backend"; + + const networkStatus = JSON.parse( + execSync(`icp network status -e ${environment} --json`, { + encoding: "utf-8", + }) + ); + const rootKey = networkStatus.root_key; + const proxyTarget = networkStatus.api_url; + + let canisterId; + try { + canisterId = execSync( + `icp canister status ${CANISTER_NAME} -e ${environment} -i`, + { encoding: "utf-8" } + ).trim(); + } catch { + console.error(` + Backend canister "${CANISTER_NAME}" not found in environment "${environment}" + + Before running the dev server, deploy the backend canister: + + icp deploy ${CANISTER_NAME} -e ${environment} + `); + process.exit(1); + } + + const server = { + headers: { + "Set-Cookie": `ic_env=${encodeURIComponent( + `PUBLIC_CANISTER_ID:${CANISTER_NAME}=${canisterId}&ic_root_key=${rootKey}` + )}; SameSite=Lax;`, + }, proxy: { - '/api': { - target: 'http://127.0.0.1:4943', - changeOrigin: true - } + "/api": { + target: proxyTarget, + changeOrigin: true, + }, }, - host: '127.0.0.1' - } + }; + + return { + plugins, + server, + }; }); diff --git a/rust/llm_chatbot/icp.yaml b/rust/llm_chatbot/icp.yaml new file mode 100644 index 0000000000..7543abcfe6 --- /dev/null +++ b/rust/llm_chatbot/icp.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://github.com/dfinity/icp-cli/raw/refs/tags/v0.1.0/docs/schemas/icp-yaml-schema.json + +canisters: + - name: llm + build: + steps: + - type: pre-built + url: https://github.com/dfinity/llm/releases/download/v0.3.1/llm-canister.wasm + sha256: 9fc6a172b13289428c6975895382c3c923fb641bcd8e8a5168469298c3cff310 + init_args: "(opt variant { ollama }, null)" + + - name: backend + recipe: + type: "@dfinity/rust@v3.3.0" + configuration: + package: backend + candid: backend/backend.did + + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.2.1" + configuration: + dir: frontend/dist + build: + - npm install --prefix frontend + - npm run build --prefix frontend + +environments: + - name: ic + network: ic + # On mainnet, the LLM canister already exists at w36hm-eqaaa-aaaal-qr76a-cai. + # Excluding it here tells icp deploy not to redeploy it on the ic network. + # Locally, icp deploy auto-injects PUBLIC_CANISTER_ID:llm into the backend + # canister, which ic-llm reads at runtime to reach the locally-deployed LLM. + canisters: + - backend + - frontend diff --git a/rust/llm_chatbot/test.sh b/rust/llm_chatbot/test.sh new file mode 100755 index 0000000000..3a03cd29a7 --- /dev/null +++ b/rust/llm_chatbot/test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +# These tests require Ollama to be running locally with the llama3.1:8b model. +# See README.md for setup instructions. + +echo "=== Test 1: prompt returns a non-empty response ===" +result=$(icp canister call backend prompt '("What is 2 + 2?")') +echo "$result" +echo "$result" | grep -qE '"[^"]+"' && echo "PASS" || (echo "FAIL: expected a quoted string response" && exit 1) + +echo "=== Test 2: chat returns a non-empty response ===" +result=$(icp canister call backend chat '(vec { variant { user = record { content = "Say hello" } } })') +echo "$result" +echo "$result" | grep -qE '"[^"]+"' && echo "PASS" || (echo "FAIL: expected a quoted string response" && exit 1)