diff --git a/packages/bcode-browser/skills/BROWSER.md b/packages/bcode-browser/skills/BROWSER.md index 21cfc1c07..ee13cae03 100644 --- a/packages/bcode-browser/skills/BROWSER.md +++ b/packages/bcode-browser/skills/BROWSER.md @@ -98,6 +98,8 @@ console.log("liveUrl for the user to watch:", liveUrl) Requires `BROWSER_USE_API_KEY` in the environment (the user should have set this before launching bcode). If absent, tell the user to get a key at https://browser-use.com and `export BROWSER_USE_API_KEY=...`. +When `BROWSER_USE_API_KEY` is set, `webfetch` is automatically enhanced with `fetch-use` (Chrome TLS fingerprint + residential proxy + session cookies) — each request is free, but consumes a small amount of proxy bandwidth from the BU account. Disable in `opencode.json` with `experimental.fetch_use: false`. + ## Attaching to a target After `connect()`, attach to a page target before driving the browser: diff --git a/packages/bcode-browser/src/fetch-use.ts b/packages/bcode-browser/src/fetch-use.ts new file mode 100644 index 000000000..83562b982 --- /dev/null +++ b/packages/bcode-browser/src/fetch-use.ts @@ -0,0 +1,59 @@ +// FetchUse — Effect service that proxies HTTP through Browser Use's fetch-use +// cloud (Chrome JA4, HTTP/2 header order, session cookies). Decisions §3.3. +// `enabled` is true when BROWSER_USE_API_KEY is set; webfetch.ts combines +// this with the user's `experimental.fetch_use` opencode.json setting. + +import { Context, Effect, Layer } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" + +const ENDPOINT = "https://fetch.browser-use.com/fetch" + +export interface FetchResult { + readonly body: ArrayBuffer + readonly contentType: string +} + +interface FetchUseRaw { + status_code: number + headers?: Record + body?: string + body_base64?: string + is_binary?: boolean + error?: string +} + +export class Service extends Context.Service Effect.Effect +}>()("@browser-use/FetchUse") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const apiKey = process.env.BROWSER_USE_API_KEY ?? "" + return Service.of({ + enabled: apiKey.length > 0, + fetch: (url, { timeoutMs }) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.post(ENDPOINT).pipe( + HttpClientRequest.setHeaders({ "Content-Type": "application/json", "X-Browser-Use-API-Key": apiKey }), + HttpClientRequest.bodyJson({ url, timeout_ms: timeoutMs }), + ) + const response = yield* HttpClient.filterStatusOk(http).execute(request) + const data = (yield* response.json) as unknown as FetchUseRaw + if (data.error) return yield* Effect.fail(new Error(`fetch-use: ${data.error}`)) + // Mirror native path's filterStatusOk: surface upstream HTTP errors as failures. + if (data.status_code >= 400) return yield* Effect.fail(new Error(`fetch-use: HTTP ${data.status_code}`)) + const body = data.is_binary && data.body_base64 + ? (new Uint8Array(Buffer.from(data.body_base64, "base64")).buffer as ArrayBuffer) + : (new TextEncoder().encode(data.body ?? "").buffer as ArrayBuffer) + const ct = + Object.entries(data.headers ?? {}).find(([k]) => k.toLowerCase() === "content-type")?.[1]?.[0] ?? "" + return { body, contentType: ct } + }).pipe(Effect.mapError((e) => (e instanceof Error ? e : new Error(String(e))))), + }) + }), +) + +export * as FetchUse from "./fetch-use" diff --git a/packages/bcode-browser/test/fetch-use.test.ts b/packages/bcode-browser/test/fetch-use.test.ts new file mode 100644 index 000000000..34f7c1148 --- /dev/null +++ b/packages/bcode-browser/test/fetch-use.test.ts @@ -0,0 +1,30 @@ +// FetchUse smoke tests. +// +// Unit: layer is constructible, `enabled` reflects BROWSER_USE_API_KEY presence. +// Live: when the key is set, end-to-end POST to fetch.browser-use.com returns +// body bytes + content-type. Skipped without the key. Config-based +// opt-out (experimental.fetch_use=false) is enforced in webfetch.ts, +// not here. + +import { expect, test } from "bun:test" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "../src/fetch-use" + +const haveKey = !!process.env.BROWSER_USE_API_KEY + +test("layer constructs and exposes `enabled` reflecting env", async () => { + const enabled = await Effect.gen(function* () { + return (yield* FetchUse.Service).enabled + }).pipe(Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), Effect.runPromise) + expect(enabled).toBe(haveKey) +}) + +test.skipIf(!haveKey)("live: fetches httpbin and returns body + content-type", async () => { + const result = await Effect.gen(function* () { + return yield* (yield* FetchUse.Service).fetch("https://httpbin.org/get", { timeoutMs: 30_000 }) + }).pipe(Effect.provide(FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer))), Effect.runPromise) + + expect(result.contentType).toContain("application/json") + expect(JSON.parse(new TextDecoder().decode(result.body)).url).toBe("https://httpbin.org/get") +}) diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 3bd070047..c3acc7c2a 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -263,6 +263,10 @@ export const Info = Schema.Struct({ mcp_timeout: Schema.optional(PositiveInt).annotate({ description: "Timeout in milliseconds for model context protocol (MCP) requests", }), + fetch_use: Schema.optional(Schema.Boolean).annotate({ + description: + "Route webfetch through the Browser Use fetch-use proxy when BROWSER_USE_API_KEY is set. Defaults to true; set false to opt out (still costs but uses native HttpClient instead).", + }), }), ), }) diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index eb1d0391e..d7e354089 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -33,6 +33,7 @@ import path from "path" import { pathToFileURL } from "url" import { Effect, Layer, Context } from "effect" import { FetchHttpClient, HttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { Ripgrep } from "../file/ripgrep" @@ -85,6 +86,7 @@ export const layer: Layer.Layer< | AppFileSystem.Service | Bus.Service | HttpClient.HttpClient + | FetchUse.Service | ChildProcessSpawner | Ripgrep.Service | Format.Service @@ -349,6 +351,7 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Instruction.defaultLayer), Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Bus.layer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(Format.defaultLayer), Layer.provide(CrossSpawnSpawner.defaultLayer), diff --git a/packages/opencode/src/tool/webfetch.ts b/packages/opencode/src/tool/webfetch.ts index 65b718c3f..82a98de43 100644 --- a/packages/opencode/src/tool/webfetch.ts +++ b/packages/opencode/src/tool/webfetch.ts @@ -1,5 +1,7 @@ import { Effect, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" +import { Config } from "@/config/config" import * as Tool from "./tool" import TurndownService from "turndown" import DESCRIPTION from "./webfetch.txt" @@ -24,6 +26,8 @@ export const WebFetchTool = Tool.define( Effect.gen(function* () { const http = yield* HttpClient.HttpClient const httpOk = HttpClient.filterStatusOk(http) + const fetchUse = yield* FetchUse.Service + const config = yield* Config.Service return { description: DESCRIPTION, @@ -47,61 +51,76 @@ export const WebFetchTool = Tool.define( const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT) - // Build Accept header based on requested format with q parameters for fallbacks - let acceptHeader = "*/*" - switch (params.format) { - case "markdown": - acceptHeader = "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" - break - case "text": - acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" - break - case "html": - acceptHeader = - "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" - break - default: - acceptHeader = - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" - } - const headers = { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", - Accept: acceptHeader, - "Accept-Language": "en-US,en;q=0.9", - } + // BrowserCode: route through fetch-use when BROWSER_USE_API_KEY is + // set and the user hasn't opted out via experimental.fetch_use=false. + const useFu = fetchUse.enabled && (yield* config.get()).experimental?.fetch_use !== false + const { arrayBuffer, contentType } = yield* (useFu + ? fetchUse + .fetch(params.url, { timeoutMs: timeout }) + .pipe(Effect.map((r) => ({ arrayBuffer: r.body, contentType: r.contentType }))) + : Effect.gen(function* () { + // Build Accept header based on requested format with q parameters for fallbacks + let acceptHeader = "*/*" + switch (params.format) { + case "markdown": + acceptHeader = + "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1" + break + case "text": + acceptHeader = "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1" + break + case "html": + acceptHeader = + "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1" + break + default: + acceptHeader = + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8" + } + const headers = { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36", + Accept: acceptHeader, + "Accept-Language": "en-US,en;q=0.9", + } - const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers)) - - // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) - const response = yield* httpOk.execute(request).pipe( - Effect.catchIf( - (err) => - err.reason._tag === "StatusCodeError" && - err.reason.response.status === 403 && - err.reason.response.headers["cf-mitigated"] === "challenge", - () => - httpOk.execute( - HttpClientRequest.get(params.url).pipe( - HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }), + const request = HttpClientRequest.get(params.url).pipe(HttpClientRequest.setHeaders(headers)) + + // Retry with honest UA if blocked by Cloudflare bot detection (TLS fingerprint mismatch) + const response = yield* httpOk.execute(request).pipe( + Effect.catchIf( + (err) => + err.reason._tag === "StatusCodeError" && + err.reason.response.status === 403 && + err.reason.response.headers["cf-mitigated"] === "challenge", + () => + httpOk.execute( + HttpClientRequest.get(params.url).pipe( + HttpClientRequest.setHeaders({ ...headers, "User-Agent": "browsercode" }), + ), + ), ), - ), - ), - Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }), - ) - - // Check content length - const contentLength = response.headers["content-length"] - if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { - throw new Error("Response too large (exceeds 5MB limit)") - } + Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.die(new Error("Request timed out")) }), + ) + + // Check content length + const contentLength = response.headers["content-length"] + if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") + } + + const arrayBuffer = yield* response.arrayBuffer + if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { + throw new Error("Response too large (exceeds 5MB limit)") + } + + return { arrayBuffer, contentType: response.headers["content-type"] || "" } + })) - const arrayBuffer = yield* response.arrayBuffer if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) { throw new Error("Response too large (exceeds 5MB limit)") } - const contentType = response.headers["content-type"] || "" const mime = contentType.split(";")[0]?.trim().toLowerCase() || "" const title = `${params.url} (${contentType})` diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index c5170f346..7156345bd 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -39,6 +39,7 @@ import { Shell } from "../../src/shell/shell" import { Snapshot } from "../../src/snapshot" import { ToolRegistry } from "@/tool/registry" import { Truncate } from "@/tool/truncate" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import * as Log from "@opencode-ai/core/util/log" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import * as Database from "../../src/storage/db" @@ -176,6 +177,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(CrossSpawnSpawner.defaultLayer), Layer.provide(Ripgrep.defaultLayer), diff --git a/packages/opencode/test/session/snapshot-tool-race.test.ts b/packages/opencode/test/session/snapshot-tool-race.test.ts index ab5a3ab7e..c615fbd08 100644 --- a/packages/opencode/test/session/snapshot-tool-race.test.ts +++ b/packages/opencode/test/session/snapshot-tool-race.test.ts @@ -50,6 +50,7 @@ import { SessionRunState } from "../../src/session/run-state" import { SessionStatus } from "../../src/session/status" import { Snapshot } from "../../src/snapshot" import { ToolRegistry } from "@/tool/registry" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Truncate } from "@/tool/truncate" import { AppFileSystem } from "@opencode-ai/core/filesystem" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" @@ -126,6 +127,7 @@ function makeHttp() { const todo = Todo.layer.pipe(Layer.provideMerge(deps)) const registry = ToolRegistry.layer.pipe( Layer.provide(Skill.defaultLayer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(CrossSpawnSpawner.defaultLayer), Layer.provide(Ripgrep.defaultLayer), diff --git a/packages/opencode/test/tool/registry.test.ts b/packages/opencode/test/tool/registry.test.ts index c33981ddf..d5448e44d 100644 --- a/packages/opencode/test/tool/registry.test.ts +++ b/packages/opencode/test/tool/registry.test.ts @@ -19,6 +19,7 @@ import { LSP } from "@/lsp/lsp" import { Instruction } from "@/session/instruction" import { Bus } from "@/bus" import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Format } from "@/format" import { Ripgrep } from "@/file/ripgrep" import * as Truncate from "@/tool/truncate" @@ -42,6 +43,7 @@ const registryLayer = ToolRegistry.layer.pipe( Layer.provide(Instruction.defaultLayer), Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Bus.layer), + Layer.provide(FetchUse.layer), Layer.provide(FetchHttpClient.layer), Layer.provide(Format.defaultLayer), Layer.provide(node), diff --git a/packages/opencode/test/tool/webfetch.test.ts b/packages/opencode/test/tool/webfetch.test.ts index 6c7f6aba7..2d8e6c389 100644 --- a/packages/opencode/test/tool/webfetch.test.ts +++ b/packages/opencode/test/tool/webfetch.test.ts @@ -2,7 +2,9 @@ import { describe, expect, test } from "bun:test" import path from "path" import { Effect, Layer } from "effect" import { FetchHttpClient } from "effect/unstable/http" +import { FetchUse } from "@browser-use/bcode-browser/fetch-use" import { Agent } from "../../src/agent/agent" +import { Config } from "@/config/config" import { Truncate } from "@/tool/truncate" import { Instance } from "../../src/project/instance" import { WithInstance } from "../../src/project/with-instance" @@ -31,7 +33,15 @@ function exec(args: { url: string; format: "text" | "markdown" | "html" }) { return WebFetchTool.pipe( Effect.flatMap((info) => info.init()), Effect.flatMap((tool) => tool.execute(args, ctx)), - Effect.provide(Layer.mergeAll(FetchHttpClient.layer, Truncate.defaultLayer, Agent.defaultLayer)), + Effect.provide( + Layer.mergeAll( + FetchUse.layer.pipe(Layer.provide(FetchHttpClient.layer)), + FetchHttpClient.layer, + Config.defaultLayer, + Truncate.defaultLayer, + Agent.defaultLayer, + ), + ), Effect.runPromise, ) }