diff --git a/.github/workflows/release-browser-loop.yml b/.github/workflows/release-browser-loop.yml index e8852076..42452b22 100644 --- a/.github/workflows/release-browser-loop.yml +++ b/.github/workflows/release-browser-loop.yml @@ -74,11 +74,11 @@ jobs: npm init -y > /dev/null npm install "$RUNNER_TEMP"/onkernel-browser-loop-*.tgz cat > smoke.mjs <<'NODE' - import { compileLoopToolCatalog, formatBrowserActResult, loop } from "@onkernel/browser-loop"; + import { compileLoopToolCatalog, executeBrowserRepl, formatBrowserActResult, loop } from "@onkernel/browser-loop"; import * as loopPi from "@onkernel/browser-loop/pi"; const { attach, createLoopModels, getLoopModel } = loopPi; - for (const [name, value] of Object.entries({ compileLoopToolCatalog, formatBrowserActResult, attach, createLoopModels, getLoopModel })) { + for (const [name, value] of Object.entries({ compileLoopToolCatalog, executeBrowserRepl, formatBrowserActResult, attach, createLoopModels, getLoopModel })) { if (typeof value !== "function") { throw new Error(`expected ${name} to be a function, got ${typeof value}`); } diff --git a/README.md b/README.md index a11a3278..3e9659a2 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ per-tool verdict. 1. **Execution layer** — `@onkernel/browser-loop` materializes the caller's exact catalog over one shared resource pool and executes canonical actions through - Kernel's computer API or a raw-CDP browser executor. + Kernel's computer, Playwright, or Browser REPL APIs, or a raw-CDP browser executor. 2. **Model layer** — `@onkernel/browser-loop/pi` opens pi-ai's whole model catalog and composes provider declarations, headers, and payload transforms around that core. Catalog compilation is declaration-only: it never sees an executor. @@ -163,7 +163,7 @@ per-tool verdict. 4. **Browser** — a Kernel cloud browser with optional profile and proxy. The model requests screenshots explicitly when it needs visual feedback. -See [`docs/architecture.md`](docs/architecture.md) for the full end-to-end flow. +See [`packages/browser-loop/docs/architecture.md`](packages/browser-loop/docs/architecture.md) for the full end-to-end flow. --- diff --git a/docs/npm-releases.md b/docs/npm-releases.md deleted file mode 100644 index 14a89730..00000000 --- a/docs/npm-releases.md +++ /dev/null @@ -1,88 +0,0 @@ -# npm releases - -`@onkernel/browser-loop` publishes from a package tag: - -- `browser-loop/v0.11.0` runs `.github/workflows/release-browser-loop.yml` - -The tag version must match `packages/browser-loop/package.json`, and the tagged -commit must be contained in `main`. - -## Trusted publishing setup - -Configure the package on npm with a GitHub Actions trusted publisher: - -| package | organization | repository | workflow filename | environment | -| --- | --- | --- | --- | --- | -| `@onkernel/browser-loop` | `kernel` | `browser-loop` | `release-browser-loop.yml` | leave blank | - -The same configuration can be created from the npm CLI: - -```sh -npm install -g npm@^11.17.0 -npm trust github @onkernel/browser-loop \ - --repo kernel/browser-loop \ - --file release-browser-loop.yml \ - --allow-publish -``` - -npm requires packages to exist before a trusted publisher can be configured, so -`@onkernel/browser-loop`'s first version is published manually — see below. - -## Releasing from a tag - -After the first manual publish, bump the version before creating the next tag: - -```sh -git checkout main -git pull --ff-only -git tag browser-loop/v0.11.1 -git push origin browser-loop/v0.11.1 -``` - -Do not create `browser-loop/v0.11.0` after manually publishing 0.11.0: npm -versions are immutable, so the workflow would attempt to republish the same -version and fail. - -## First publish of the package - -npm requires a package to exist before a trusted publisher can be configured for -it. The first `@onkernel/browser-loop` release is therefore a manual publish -from a maintainer account in the `onkernel` npm organization. - -Rename the GitHub repository to `kernel/browser-loop` before publishing. Trusted -publishers bind to a *(repository, workflow filename)* pair, so doing the repo -rename, first publish, and trusted-publisher setup together avoids configuring -that pair twice. - -Publish the same tarball that passed the install smoke test: - -```sh -git clone https://github.com/kernel/browser-loop.git -cd browser-loop -git checkout main -git pull --ff-only - -npm ci -npm run build --workspace @onkernel/browser-loop -npm run typecheck --workspace @onkernel/browser-loop -npm test --workspace @onkernel/browser-loop - -PACK_DIR=$(mktemp -d) -npm pack --workspace @onkernel/browser-loop --pack-destination "$PACK_DIR" -TARBALL="$PACK_DIR/onkernel-browser-loop-0.11.0.tgz" - -SMOKE_DIR=$(mktemp -d) -( - cd "$SMOKE_DIR" - npm init -y > /dev/null - npm install "$TARBALL" - node --input-type=module -e \ - 'import("@onkernel/browser-loop").then(m => console.log(Object.keys(m)))' -) - -npm login -npm publish "$TARBALL" --access public -``` - -Then configure the trusted publisher using the command above or the npm package -page (Settings → Publishing access → Add trusted publisher). diff --git a/packages/browser-loop/CHANGELOG.md b/packages/browser-loop/CHANGELOG.md index 05858056..5849d5ee 100644 --- a/packages/browser-loop/CHANGELOG.md +++ b/packages/browser-loop/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Add `loop.tools.repl()` (`kloop.repl.v1`) as a wrapper around Kernel browsers' persistent JavaScript API, including ordered text/image output and explicit REPL lifecycle metadata. +- Add the framework-neutral `executeBrowserRepl()` client and the pi extension's `repl` selector. + ## 0.12.1 - 2026-09-04 - Relax the `@onkernel/sdk` dependency from an exact pin to `>=0.49.0 <1.0.0`. diff --git a/packages/browser-loop/README.md b/packages/browser-loop/README.md index 1b393a44..96dbcca0 100644 --- a/packages/browser-loop/README.md +++ b/packages/browser-loop/README.md @@ -9,7 +9,7 @@ binding; Eve and AI SDK are anticipated next. | import | what it is | | --- | --- | -| `@onkernel/browser-loop` | The framework-neutral core: canonical actions, the tool namespace, catalog compilation, the tool menu, and Kernel-browser execution. Core declarations (`LoopToolDeclaration`) and executables (`LoopExecutableTool`) import nothing from pi — schemas come from `typebox` directly — and a unit test enforces the boundary. | +| `@onkernel/browser-loop` | The framework-neutral core: canonical actions, the tool namespace, catalog compilation, Kernel-browser execution, and the persistent Browser REPL API wrapper. Core declarations (`LoopToolDeclaration`) and executables (`LoopExecutableTool`) import nothing from pi, and a unit test enforces the boundary. | | `@onkernel/browser-loop/pi` | The pi binding: `attach()`, model resolution, transport derivation, provider adapters, and provider retry. | Installing the package into pi (`pi install npm:@onkernel/browser-loop`) registers the @@ -302,10 +302,28 @@ loop.tools.computer.batch({ actions: ["click", "keypress", "screenshot"] }); loop.tools.browser.batch({ actions: ["snapshot", "click", "wait_for", "text"] }); loop.tools.playwright(); +loop.tools.repl(); ``` A toolset carries that surface's primitives, not every tool it has: `browser_act`, -`computer_zoom`, and the batch forms are selected explicitly. +`computer_zoom`, the persistent Browser REPL, and the batch forms are selected explicitly. + +`loop.tools.repl()` wraps the Kernel browser VM's `POST /repl` API. Top-level bindings +survive across calls, output remains explicitly ordered, and the VM runtime exposes +raw CDP/browser helpers, accessibility snapshots, WebMCP, `repl.help()`, and optional dynamic +imports of Patchright or Playwright: + +```ts +const compiled = kb.compile({ + model: "anthropic:claude-opus-5", + tools: [loop.tools.repl()], +}); + +// compiled.agentTools is directly usable by pi's Agent. +``` + +See [Architecture: Browser REPL API wrapper](docs/architecture.md#browser-repl-api-wrapper) +for the ownership and result-mapping boundary. Batches are mechanical primitive lists. They have no branching, saved values, references, or workflow DSL. @@ -495,7 +513,7 @@ pi -p --provider openai --model gpt-5.6-sol \ ### The menu -Eight entries, one per capability. Availability is per model, and `/browser-tools` +Nine entries, one per capability. Availability is per model, and `/browser-tools` tells you which apply to the one you selected. | entry | tools | works on | @@ -504,6 +522,7 @@ tells you which apply to the one you selected. | `computer` | canonical computer primitives plus `computer_batch` | every provider | | `browser-act` | `browser_act`, the verified-plan tool | every provider except Moonshot, which rejects its schema size | | `playwright` | `playwright_execute` | every provider | +| `repl` | persistent `browser_repl` JavaScript cells | every provider | | `anthropic-computer` | Anthropic's native computer tool | Anthropic models with that native surface | | `anthropic-browser` | Anthropic's native browser tool | Anthropic models with that native surface | | `openai-computer` | OpenAI's native computer tool | OpenAI models with that native surface | @@ -570,7 +589,8 @@ npm run example:harness --workspace @onkernel/browser-loop -- \ ``` See [`examples/`](examples) for direct catalog/model usage, these parameterized -agent and harness examples, and the Anthropic-native composition. +agent and harness examples, and the Anthropic-native composition. Release steps +are documented in [`docs/npm-releases.md`](docs/npm-releases.md). ## License diff --git a/docs/architecture.md b/packages/browser-loop/docs/architecture.md similarity index 84% rename from docs/architecture.md rename to packages/browser-loop/docs/architecture.md index 95511305..1ecb95ec 100644 --- a/docs/architecture.md +++ b/packages/browser-loop/docs/architecture.md @@ -91,7 +91,8 @@ The main groups are: - `loop.tools.browser.*`: CDP/page tools, using element refs and viewport pixels. - `loop.tools.computer.*`: Kernel OS input/read tools, using pixel coordinates by default. -- `loop.tools.playwright()`: a Playwright code execution tool. +- `loop.tools.playwright()`: a wrapper around Kernel's Playwright execution API. +- `loop.tools.repl()`: a wrapper around Kernel's persistent Browser REPL API. - `loop.toolsets.browser()`, `computer()`, and `mixed()`: ordinary convenience arrays of Browser Loop-authored tools. - `loop.providers.*`: only provider-native tools and predefined toolsets backed @@ -118,8 +119,8 @@ await compiled.apply(harness); Nothing mutates in place: a change compiles a new pair, and `compile()` throws before anything reaches pi. Existing tool -identity with a changed schema, executor, or coordinates counts as a real -replacement. Additions made from inside a running tool are recorded in pi's +identity with a changed model-facing name, schema, or coordinate contract counts +as a real replacement. Additions made from inside a running tool are recorded in pi's Anthropic-compatible `addedToolNames` marker only when that provider/model can defer ordinary function tools. Additions outside a tool call are eager. Provider-native tools are always eager. @@ -136,12 +137,41 @@ catalog and model changes. It owns: - one canonical computer translator; - one lazily created raw-CDP `BrowserExecutor`; - browser element-ref and frame state; -- screenshot and Playwright execution capabilities. +- screenshot, Playwright, and persistent Browser REPL execution capabilities. Recompiling and applying a catalog preserves refs, tabs, browser state, and caches. Tools are materialized as small adapters over that shared pool, exactly once per spec object. +## Browser REPL API wrapper + +`loop.tools.repl()` is the ordinary function tool `browser_repl`, with stable +identity `kloop.repl.v1`. It is framework-neutral: `LoopExecutionResources` +materializes it like every other Loop spec, the pi binding adapts that executable +to an `AgentTool`, and future framework bindings can adapt the same executable. +The root-exported `executeBrowserRepl()` is available when a binding needs the +HTTP client directly. + +Browser Loop does not contain, publish, or start the REPL daemon. The browser VM +owns persistent JavaScript state and kernel-images owns `POST /repl`, daemon +packaging, process supervision, reset and destructive-timeout semantics, CDP and +browser helpers, WebMCP, and optional Patchright/Playwright installation. +Consequently this tool supports Kernel browser handles whose image implements +`POST /repl`; it has no local-Chrome execution mode. + +The HTTP client requires the browser's `base_url` and `cdp_ws_url`. It copies the +existing `jwt` query parameter from the CDP URL onto `/repl`, forwards +the execution's abort signal, and sends `{code, timeout_sec?, reset?}`. It +validates local input constraints before dispatch and rejects non-2xx or malformed +responses. + +The API response keeps its exact `repl_id`, error and stack, duration, +`content_truncated`, and `repl_terminated` state. `LoopExecutionResources` maps +response items in order: `write` text remains plain, `stdout` and `stderr` keep a +channel prefix, and images retain their MIME type and base64 data. A failed REPL +execution becomes model-readable output with `details.isError`; transport and +protocol failures throw as tool failures. + ## Action planes and result feedback Canonical actions live under `packages/browser-loop/src/core/actions/`: @@ -158,6 +188,7 @@ Tools return only the result requested by the model: - Read actions return their requested text or structured data. - Screenshot and zoom actions return images. - `browser_act` returns causal outcomes and a bounded successor diff. +- `browser_repl` preserves ordered text/image output and process-lifecycle metadata. - Failed batches replace images captured by earlier explicit screenshot steps with textual markers. @@ -215,8 +246,9 @@ transport. ### The tool menu -`loopToolMenu(model, selected)` in `packages/browser-loop/src/core/menu.ts` returns every tool -Browser Loop can offer for a model, each marked available or not. It decides availability +`loopToolMenu(model, selected)` in `packages/browser-loop/src/core/menu.ts` +returns every tool Browser Loop can offer for a model, each marked available or +not. It decides availability by compiling the candidate catalog rather than by restating the compiler's rules, so the menu cannot drift from what `compileLoopToolCatalog` accepts: an entry is available exactly when selecting it compiles. Compilation is pure and @@ -280,7 +312,7 @@ user prompt -> provider stream -> incoming native/function call normalization -> shared LoopExecutionResources - -> Kernel computer API or raw-CDP BrowserExecutor + -> Kernel computer/Playwright/REPL API or raw-CDP BrowserExecutor -> policy-specific action result -> transcript + TUI/stdout/JSONL ``` @@ -289,7 +321,10 @@ user prompt - `packages/browser-loop/test/tool-catalog.test.ts`: identities, collisions, provider composition, compatibility, declarations, and coordinate contracts. -- `packages/browser-loop/test/resources.test.ts`: action feedback and batch boundaries. +- `packages/browser-loop/test/resources.test.ts`: action feedback, Browser REPL + result mapping, and batch boundaries. +- `packages/browser-loop/test/repl-client.test.ts` and `repl-tool.test.ts`: + Browser REPL authentication, validation, declaration, and catalog behavior. - `packages/browser-loop/test/attach.test.ts` and `attach-session.test.ts`: compiled pairs, applying one to a running harness, and the behaviors `activate()` installs. diff --git a/packages/browser-loop/docs/npm-releases.md b/packages/browser-loop/docs/npm-releases.md new file mode 100644 index 00000000..0d3c60f3 --- /dev/null +++ b/packages/browser-loop/docs/npm-releases.md @@ -0,0 +1,64 @@ +# npm releases + +`@onkernel/browser-loop` publishes from tags matching `browser-loop/v*` through +`.github/workflows/release-browser-loop.yml`. + +## Release contract + +Before pushing a tag: + +1. Update `packages/browser-loop/package.json` and `package-lock.json` to the + release version. +2. Merge that version change to `main`. +3. Tag the merged commit as `browser-loop/v`. + +The workflow rejects a tag when its commit is not contained in `main` or when +the tag version differs from `packages/browser-loop/package.json`. + +```sh +git checkout main +git pull --ff-only +git tag browser-loop/v0.13.0 +git push origin browser-loop/v0.13.0 +``` + +npm versions and Git tags are immutable. Never reuse a failed or previously +published version; fix the problem, bump again, and create a new tag. + +## What the workflow validates + +The release job uses Node 24 and npm 11, then runs: + +```sh +npm run check:lockfile +npm ci +npm run build --workspace @onkernel/browser-loop +npm run typecheck --workspace @onkernel/browser-loop +npm test --workspace @onkernel/browser-loop +npm pack --workspace @onkernel/browser-loop +``` + +It installs the packed tarball into a fresh project and verifies both package +entry points before publishing the same package to npm. + +## Trusted publishing + +The npm package is configured for GitHub Actions trusted publishing: + +| package | organization | repository | workflow filename | environment | +| --- | --- | --- | --- | --- | +| `@onkernel/browser-loop` | `kernel` | `browser-loop` | `release-browser-loop.yml` | none | + +To restore that configuration if it is removed: + +```sh +npm install -g npm@^11.17.0 +npm trust github @onkernel/browser-loop \ + --repo kernel/browser-loop \ + --file release-browser-loop.yml \ + --allow-publish +``` + +The workflow receives `id-token: write` and publishes with +`npm publish --workspace @onkernel/browser-loop --access public`; it does not use +a long-lived npm token. diff --git a/packages/browser-loop/examples/repl-agent.ts b/packages/browser-loop/examples/repl-agent.ts new file mode 100644 index 00000000..92d890c3 --- /dev/null +++ b/packages/browser-loop/examples/repl-agent.ts @@ -0,0 +1,27 @@ +import { Agent } from "@earendil-works/pi-agent-core"; +import Kernel from "@onkernel/sdk"; +import { loop } from "../src/index"; +import { attach } from "../src/pi/index"; + +const client = new Kernel({ apiKey: process.env.KERNEL_API_KEY! }); +const browser = await client.browsers.create({ stealth: true, timeout_seconds: 600 }); +const handle = attach({ client, browser }); +const compiled = handle.compile({ + model: "anthropic:claude-opus-5", + tools: [loop.tools.repl()], +}); +const agent = new Agent({ + streamFn: (model, context, options) => compiled.models.streamSimple(model, context, options), + initialState: { + model: compiled.model, + tools: [...compiled.agentTools], + systemPrompt: "Use browser_repl. Persist useful JavaScript helpers between calls and emit answers with repl.write().", + }, +}); + +try { + await agent.prompt("Open example.com, save a helper that reads the page title, then call it and report the title."); +} finally { + await handle.dispose(); + await client.browsers.deleteByID(browser.session_id); +} diff --git a/packages/browser-loop/src/core/menu.ts b/packages/browser-loop/src/core/menu.ts index 5335159d..546a5cb6 100644 --- a/packages/browser-loop/src/core/menu.ts +++ b/packages/browser-loop/src/core/menu.ts @@ -3,7 +3,7 @@ import { loop } from "./tools"; import { compileLoopToolCatalog, type LoopToolSpec } from "./tool-catalog"; /** Where a menu entry comes from, for grouping in a picker. */ -export type LoopToolMenuGroup = "browser" | "computer" | "playwright" | "native"; +export type LoopToolMenuGroup = "browser" | "computer" | "playwright" | "repl" | "native"; /** One offerable item: a single tool, or a native toolset selected as a unit. */ export interface LoopToolMenuEntry { @@ -95,6 +95,7 @@ function offerableEntries(): OfferableEntry[] { entries.push(single(tool, "computer")); } entries.push(single(loop.tools.playwright(), "playwright")); + entries.push(single(loop.tools.repl(), "repl")); entries.push(single(loop.providers.openai.tools.computer(), "native")); entries.push(single(loop.providers.anthropic.tools.computer(), "native")); entries.push(single(loop.providers.anthropic.tools.browser(), "native")); diff --git a/packages/browser-loop/src/core/repl.ts b/packages/browser-loop/src/core/repl.ts new file mode 100644 index 00000000..22cc8ee2 --- /dev/null +++ b/packages/browser-loop/src/core/repl.ts @@ -0,0 +1,127 @@ +export type BrowserReplTextChannel = "write" | "stdout" | "stderr"; + +export interface BrowserReplTextContent { + type: "text"; + channel: BrowserReplTextChannel; + text: string; +} + +export interface BrowserReplImageContent { + type: "image"; + mime_type: string; + data_b64: string; +} + +export type BrowserReplContent = BrowserReplTextContent | BrowserReplImageContent; + +export interface BrowserReplRequest { + code: string; + timeout_sec?: number; + reset?: boolean; +} + +export interface BrowserReplResult { + success: boolean; + repl_id: string; + error?: string; + stack?: string; + content?: BrowserReplContent[]; + content_truncated?: boolean; + repl_terminated?: boolean; + duration_ms?: number; +} + +export interface BrowserReplEndpoint { + base_url?: string; + cdp_ws_url?: string; +} + +export interface ExecuteBrowserReplOptions { + signal?: AbortSignal; + fetch?: typeof fetch; +} + +/** Execute one cell through the Browser REPL owned by a Kernel browser VM. */ +export async function executeBrowserRepl( + browser: BrowserReplEndpoint, + request: BrowserReplRequest, + options: ExecuteBrowserReplOptions = {}, +): Promise { + validateRequest(request); + const endpoint = authenticatedReplUrl(browser); + const response = await (options.fetch ?? fetch)(endpoint, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + signal: options.signal, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`Browser REPL request failed (${response.status}): ${responseError(body)}`); + } + const result: unknown = await response.json(); + if (!isBrowserReplResult(result)) throw new Error("Browser REPL returned an invalid response"); + return result; +} + +function authenticatedReplUrl(browser: BrowserReplEndpoint): URL { + if (!browser.base_url) throw new Error("browser has no base_url; Browser REPL is unavailable"); + if (!browser.cdp_ws_url) throw new Error("browser has no cdp_ws_url; Browser REPL authentication is unavailable"); + const cdpUrl = new URL(browser.cdp_ws_url); + const jwt = cdpUrl.searchParams.get("jwt"); + if (!jwt) throw new Error("browser cdp_ws_url has no jwt; Browser REPL authentication is unavailable"); + const endpoint = new URL(`${browser.base_url.replace(/\/+$/, "")}/repl`); + endpoint.searchParams.set("jwt", jwt); + return endpoint; +} + +function validateRequest(request: BrowserReplRequest): void { + if (typeof request.code !== "string") throw new TypeError("Browser REPL code must be a string"); + if (request.code.length === 0 && request.reset !== true) throw new Error("Browser REPL code may be empty only when reset is true"); + if (request.timeout_sec !== undefined && (!Number.isInteger(request.timeout_sec) || request.timeout_sec < 1 || request.timeout_sec > 300)) { + throw new RangeError("Browser REPL timeout_sec must be an integer between 1 and 300"); + } + if (request.reset !== undefined && typeof request.reset !== "boolean") throw new TypeError("Browser REPL reset must be a boolean"); +} + +function isBrowserReplResult(value: unknown): value is BrowserReplResult { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const result = value as Record; + return typeof result.success === "boolean" + && typeof result.repl_id === "string" + && optionalType(result.error, "string") + && optionalType(result.stack, "string") + && optionalType(result.content_truncated, "boolean") + && optionalType(result.repl_terminated, "boolean") + && optionalType(result.duration_ms, "number") + && (result.content === undefined || (Array.isArray(result.content) && result.content.every(isBrowserReplContent))); +} + +function isBrowserReplContent(value: unknown): value is BrowserReplContent { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const item = value as Record; + if (item.type === "text") { + return (item.channel === "write" || item.channel === "stdout" || item.channel === "stderr") + && typeof item.text === "string"; + } + return item.type === "image" + && typeof item.mime_type === "string" + && item.mime_type.startsWith("image/") + && typeof item.data_b64 === "string"; +} + +function optionalType(value: unknown, type: "boolean" | "number" | "string"): boolean { + return value === undefined || typeof value === type; +} + +function responseError(body: string): string { + try { + const parsed: unknown = JSON.parse(body); + if (parsed && typeof parsed === "object" && typeof (parsed as { message?: unknown }).message === "string") { + return (parsed as { message: string }).message; + } + } catch { + // Use the bounded response body below. + } + return body.slice(0, 8_192) || "empty response"; +} diff --git a/packages/browser-loop/src/core/resources.ts b/packages/browser-loop/src/core/resources.ts index b8425372..6254cdad 100644 --- a/packages/browser-loop/src/core/resources.ts +++ b/packages/browser-loop/src/core/resources.ts @@ -5,6 +5,11 @@ import { formatBrowserActResult } from "./browser-result-format"; import { BatchExecutionError, InternalComputerTranslator, type KernelBrowser, type PlaywrightExecutionResult } from "./translator/translator"; import type { BrowserExecutor } from "./translator/browser"; import type { BatchExecutionResult, BatchReadResult, BrowserState, BrowserWaitForResult } from "./translator/types"; +import { + executeBrowserRepl, + type BrowserReplRequest, + type BrowserReplResult, +} from "./repl"; /** Structured execution metadata returned by materialized Loop tools. */ export interface LoopExecutionDetails { @@ -18,6 +23,11 @@ export interface LoopExecutionDetails { stdout?: string; stderr?: string; error?: string; + stack?: string; + replId?: string; + replTerminated?: boolean; + contentTruncated?: boolean; + durationMs?: number; } /** One content block returned to the model by a materialized Loop tool. */ @@ -72,6 +82,7 @@ export class LoopExecutionResources { spec, execute: async (input, signal) => { if (spec.execution.kind === "playwright") return this.executePlaywright(spec.name, input); + if (spec.execution.kind === "repl") return this.executeRepl(spec.name, input, signal); const actions = spec.execution.toActions(input); return this.executeActions(spec, actions, signal); }, @@ -92,6 +103,10 @@ export class LoopExecutionResources { return this.translator.executePlaywright(code, timeoutSec); } + async repl(request: BrowserReplRequest, signal?: AbortSignal): Promise { + return executeBrowserRepl(this.browser, request, { signal }); + } + async dispose(): Promise { this.translator.dispose(); } @@ -138,6 +153,41 @@ export class LoopExecutionResources { }; } + private async executeRepl(name: string, input: unknown, signal?: AbortSignal): Promise { + const parameters = asRecord(input); + const request: BrowserReplRequest = { + code: requireParameter(parameters.code, "string", `${name} requires string code`), + ...(parameters.timeout_sec !== undefined ? { timeout_sec: requireFiniteNumber(parameters.timeout_sec, `${name} timeout_sec must be a number`) } : {}), + ...(parameters.reset !== undefined ? { reset: requireParameter(parameters.reset, "boolean", `${name} reset must be a boolean`) } : {}), + }; + try { + const result = await this.repl(request, signal); + const content: ToolContent = (result.content ?? []).map((item) => item.type === "image" + ? { type: "image", data: item.data_b64, mimeType: item.mime_type } + : { type: "text", text: item.channel === "write" ? item.text : `${item.channel}: ${item.text}` }); + if (!result.success) content.push({ type: "text", text: `error: ${result.error ?? "Browser REPL execution reported failure"}` }); + const statusText = result.success + ? `Browser REPL ${result.repl_id} executed successfully.` + : `Browser REPL ${result.repl_id} failed: ${result.error ?? "unknown error"}`; + if (content.length === 0) content.push({ type: "text", text: statusText }); + return { + content, + details: { + statusText, + replId: result.repl_id, + replTerminated: result.repl_terminated ?? false, + contentTruncated: result.content_truncated ?? false, + ...(result.duration_ms !== undefined ? { durationMs: result.duration_ms } : {}), + ...(result.error ? { error: result.error } : {}), + ...(result.stack ? { stack: result.stack } : {}), + ...(!result.success ? { isError: true } : {}), + }, + }; + } catch (error) { + throw new Error(`${name} failed: ${errorMessage(error)}`, { cause: error }); + } + } + private async executePlaywright(name: string, input: unknown): Promise { const parameters = asRecord(input); const code = parameters.code; @@ -248,6 +298,16 @@ function formatValue(value: unknown): string { return typeof value === "string" ? value : JSON.stringify(value); } +function requireParameter(value: unknown, type: "string" | "boolean", error: string): T { + if (typeof value !== type) throw new Error(error); + return value as T; +} + +function requireFiniteNumber(value: unknown, error: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(error); + return value; +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/browser-loop/src/core/tool-catalog.ts b/packages/browser-loop/src/core/tool-catalog.ts index 929f1445..5cf528b9 100644 --- a/packages/browser-loop/src/core/tool-catalog.ts +++ b/packages/browser-loop/src/core/tool-catalog.ts @@ -43,7 +43,8 @@ export type LoopToolExecution = /** Block later calls in the same assistant turn after this tool fails. */ readonly stopTurnOnFailureMessage?: string; } - | { readonly kind: "playwright" }; + | { readonly kind: "playwright" } + | { readonly kind: "repl" }; export type LoopProviderBinding = | { diff --git a/packages/browser-loop/src/core/tools.ts b/packages/browser-loop/src/core/tools.ts index 74273eb9..fe1ed743 100644 --- a/packages/browser-loop/src/core/tools.ts +++ b/packages/browser-loop/src/core/tools.ts @@ -298,6 +298,26 @@ function playwright(options: LoopToolNameOptions = {}): LoopToolSpec { }); } +function repl(options: LoopToolNameOptions = {}): LoopToolSpec { + return createSpec({ + identity: "kloop.repl.v1", + preferredName: "browser_repl", + name: options.name, + origin: "loop", + declaration: { + name: "browser_repl", + description: "Execute JavaScript in the browser VM's persistent REPL. Top-level bindings survive across calls; browser, CDP, WebMCP, Node, and optional Patchright/Playwright helpers are available. Emit ordered output with repl.write, console methods, or repl.emitImage.", + parameters: Type.Object({ + code: Type.String({ description: "JavaScript to evaluate. May be empty only when reset is true." }), + timeout_sec: Type.Optional(Type.Integer({ minimum: 1, maximum: 300, default: 60 })), + reset: Type.Optional(Type.Boolean({ description: "Replace the current REPL before evaluating code." })), + }, { additionalProperties: false }), + }, + execution: { kind: "repl" }, + stateMutating: true, + }); +} + function anthropicNativeComputer(options: { version: "20260801"; zoom?: boolean } = { version: "20260801" }): LoopToolSpec { if (options.version !== "20260801") throw new Error(`unsupported Anthropic native computer version "${String(options.version)}"`); const declaration = { @@ -773,7 +793,7 @@ const providers = Object.freeze({ */ export const loop = Object.freeze({ coordinates: Object.freeze({ pixels: () => pixels, normalized }), - tools: Object.freeze({ browser: browserTools, computer: computerTools, playwright }), + tools: Object.freeze({ browser: browserTools, computer: computerTools, playwright, repl }), toolsets: Object.freeze({ browser: browserToolset, computer: computerToolset, diff --git a/packages/browser-loop/src/index.ts b/packages/browser-loop/src/index.ts index 26dcf158..fcedd9bf 100644 --- a/packages/browser-loop/src/index.ts +++ b/packages/browser-loop/src/index.ts @@ -4,6 +4,17 @@ export * from "./core/model-info"; export * from "./core/tool-catalog"; export * from "./core/tools"; export { normalizeGotoUrl } from "./core/url"; +export { + executeBrowserRepl, + type BrowserReplContent, + type BrowserReplEndpoint, + type BrowserReplImageContent, + type BrowserReplRequest, + type BrowserReplResult, + type BrowserReplTextChannel, + type BrowserReplTextContent, + type ExecuteBrowserReplOptions, +} from "./core/repl"; // pi-flavored entry points: these shadow the star exports above, keeping // provider-qualified model refs ("openai:gpt-5.5") and pi-supplied model diff --git a/packages/browser-loop/src/pi-extension/selection.ts b/packages/browser-loop/src/pi-extension/selection.ts index 0fcce0ad..1b4423f9 100644 --- a/packages/browser-loop/src/pi-extension/selection.ts +++ b/packages/browser-loop/src/pi-extension/selection.ts @@ -40,6 +40,7 @@ const MENU: Readonly LoopToolS ], "browser-act": () => [loop.tools.browser.act()], playwright: () => [loop.tools.playwright()], + repl: () => [loop.tools.repl()], "anthropic-computer": () => [loop.providers.anthropic.tools.computer({ version: "20260801", zoom: true })], "anthropic-browser": () => [loop.providers.anthropic.tools.browser({ version: "20260801", javascript: true })], "openai-computer": () => [loop.providers.openai.tools.computer()], diff --git a/packages/browser-loop/test/menu.test.ts b/packages/browser-loop/test/menu.test.ts index 5038a9ad..2a33c841 100644 --- a/packages/browser-loop/test/menu.test.ts +++ b/packages/browser-loop/test/menu.test.ts @@ -80,8 +80,9 @@ describe("loopToolMenu", () => { it("covers the whole offerable surface, grouped", () => { const menu = loopToolMenu("openai:gpt-5.5"); const groups = new Set(menu.map((entry) => entry.group)); - expect(groups).toEqual(new Set(["browser", "computer", "playwright", "native"])); + expect(groups).toEqual(new Set(["browser", "computer", "playwright", "repl", "native"])); expect(menu.some((entry) => entry.label === "playwright_execute")).toBe(true); + expect(menu.some((entry) => entry.label === "browser_repl")).toBe(true); expect(menu.some((entry) => entry.label === "browser_act")).toBe(true); expect(menu.every((entry) => entry.tools.length > 0)).toBe(true); }); diff --git a/packages/browser-loop/test/repl-client.test.ts b/packages/browser-loop/test/repl-client.test.ts new file mode 100644 index 00000000..7c78365c --- /dev/null +++ b/packages/browser-loop/test/repl-client.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from "vitest"; +import { executeBrowserRepl } from "../src/core/repl"; + +const browser = { + base_url: "https://metro.example/browser/kernel/", + cdp_ws_url: "wss://metro.example/browser/cdp?jwt=secret-token", +}; + +describe("executeBrowserRepl", () => { + it("authenticates through the browser JWT and preserves the response", async () => { + const fetch = vi.fn(async () => new Response(JSON.stringify({ + success: true, + repl_id: "repl_1", + content: [{ type: "text", channel: "write", text: "ok" }], + }), { status: 200, headers: { "content-type": "application/json" } })); + const result = await executeBrowserRepl(browser, { code: "repl.write('ok')" }, { fetch }); + expect(result.repl_id).toBe("repl_1"); + const [url, init] = fetch.mock.calls[0]!; + expect(String(url)).toBe("https://metro.example/browser/kernel/repl?jwt=secret-token"); + expect(init).toMatchObject({ method: "POST", body: JSON.stringify({ code: "repl.write('ok')" }) }); + }); + + it("rejects requests locally before dispatch", async () => { + const fetch = vi.fn(); + await expect(executeBrowserRepl(browser, { code: "", timeout_sec: 0 }, { fetch })).rejects.toThrow(/empty only when reset/); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("rejects malformed typed content", async () => { + const fetch = vi.fn(async () => new Response(JSON.stringify({ + success: true, + repl_id: "repl_1", + content: [{ type: "text", channel: "unknown", text: "bad" }], + }), { status: 200 })); + await expect(executeBrowserRepl(browser, { code: "1" }, { fetch })).rejects.toThrow("invalid response"); + }); + + it("surfaces structured HTTP errors", async () => { + const fetch = vi.fn(async () => new Response(JSON.stringify({ message: "REPL unavailable" }), { status: 500 })); + await expect(executeBrowserRepl(browser, { code: "1" }, { fetch })).rejects.toThrow("Browser REPL request failed (500): REPL unavailable"); + }); +}); diff --git a/packages/browser-loop/test/repl-tool.test.ts b/packages/browser-loop/test/repl-tool.test.ts new file mode 100644 index 00000000..edca670a --- /dev/null +++ b/packages/browser-loop/test/repl-tool.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { compileLoopToolCatalog, loop } from "../src/index"; + +const model = { provider: "test", id: "test-model", api: "test" }; + +describe("Browser REPL tool", () => { + it("has one stable identity and the POST /repl input contract", () => { + const tool = loop.tools.repl(); + expect(tool.identity).toBe("kloop.repl.v1"); + expect(tool.name).toBe("browser_repl"); + expect(tool.execution).toEqual({ kind: "repl" }); + expect(tool.declaration.parameters).toMatchObject({ + type: "object", + required: ["code"], + additionalProperties: false, + properties: { + code: { type: "string" }, + timeout_sec: { type: "integer", minimum: 1, maximum: 300, default: 60 }, + reset: { type: "boolean" }, + }, + }); + }); + + it("compiles as an ordinary framework-neutral function tool", () => { + const catalog = compileLoopToolCatalog({ model, requestedTools: [loop.tools.repl()] }); + expect(catalog.entries).toMatchObject([{ identity: "kloop.repl.v1", name: "browser_repl", transport: "function" }]); + }); +}); diff --git a/packages/browser-loop/test/resources.test.ts b/packages/browser-loop/test/resources.test.ts index 65746dd5..09f70ce0 100644 --- a/packages/browser-loop/test/resources.test.ts +++ b/packages/browser-loop/test/resources.test.ts @@ -6,7 +6,8 @@ import type { BatchReadResult } from "../src/core/translator/types"; const browser = { session_id: "browser_123", - cdp_ws_url: "wss://example.test/cdp", + base_url: "https://example.test/browser/kernel", + cdp_ws_url: "wss://example.test/cdp?jwt=test-token", viewport: { width: 1440, height: 900 }, } as KernelBrowser; @@ -187,6 +188,42 @@ describe("LoopExecutionResources results and batch boundaries", () => { expect(browserScreenshot).not.toHaveBeenCalled(); }); + it("returns ordered Browser REPL text and image output with lifecycle details", async () => { + const fetch = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(new Response(JSON.stringify({ + success: false, + repl_id: "repl_1", + error: "boom", + repl_terminated: true, + content_truncated: true, + duration_ms: 42, + content: [ + { type: "text", channel: "stdout", text: "before" }, + { type: "image", mime_type: "image/png", data_b64: "aW1hZ2U=" }, + { type: "text", channel: "write", text: "after" }, + ], + }), { status: 200 })); + try { + const { resources } = setup(); + const result = await resources.materialize(loop.tools.repl()).execute({ code: "run()" }); + expect(result.content).toEqual([ + { type: "text", text: "stdout: before" }, + { type: "image", data: "aW1hZ2U=", mimeType: "image/png" }, + { type: "text", text: "after" }, + { type: "text", text: "error: boom" }, + ]); + expect(result.details).toMatchObject({ + isError: true, + replId: "repl_1", + replTerminated: true, + contentTruncated: true, + durationMs: 42, + }); + expect(String(fetch.mock.calls[0]![0])).toBe("https://example.test/browser/kernel/repl?jwt=test-token"); + } finally { + fetch.mockRestore(); + } + }); + it("keeps Playwright execution failures as model-readable content", async () => { const { resources } = setup({ failPlaywright: true }); const result = await resources.materialize(loop.tools.playwright()).execute({ code: "throw new Error('boom')" }); diff --git a/packages/browser-loop/test/selection.test.ts b/packages/browser-loop/test/selection.test.ts index 302f7008..69b6be63 100644 --- a/packages/browser-loop/test/selection.test.ts +++ b/packages/browser-loop/test/selection.test.ts @@ -52,12 +52,13 @@ describe("Loop pi selectors", () => { "computer_batch", ]); }); - it("offers exactly the eight menu entries", () => { + it("offers exactly the nine menu entries", () => { expect([...LOOP_SELECTORS]).toEqual([ "browser", "computer", "browser-act", "playwright", + "repl", "anthropic-computer", "anthropic-browser", "openai-computer",