diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e2286..4016845 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,12 +155,17 @@ jobs: shellcheck scripts/build-lambda-microvm-artifact.sh - name: Validate runner Dockerfile - run: >- - docker buildx build --check - --platform linux/arm64 - --target lambda-microvm-runner - -f api/Dockerfile - . + run: | + docker buildx build --check \ + --platform linux/arm64 \ + --target lambda-microvm-runner \ + -f api/Dockerfile \ + . + docker buildx build --check \ + --platform linux/arm64 \ + --target lambda-microvm-app-host \ + -f api/Dockerfile \ + . - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: diff --git a/api/Dockerfile b/api/Dockerfile index f8d6713..0133e3d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -146,7 +146,8 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \ 2>/dev/null || true COPY api/src/entrypoint.sh ./entrypoint.sh -RUN chmod +x ./entrypoint.sh +COPY api/src/hosted-app-launcher.sh /usr/local/bin/codeapi-hosted-app-launcher +RUN chmod +x ./entrypoint.sh /usr/local/bin/codeapi-hosted-app-launcher # ============================================================================ # Stage 2b: AWS Lambda MicroVM container base image @@ -169,6 +170,17 @@ ENV PORT=8080 \ EXPOSE 8080/tcp ENTRYPOINT ["/sandbox_api/entrypoint.sh"] +# Dedicated resident-process host. This image deliberately keeps the runner +# control listener on 8080 and exposes one fixed user-app port separately. +# `/execute` is disabled when hosted-app mode is enabled; the control plane +# restores a checkpoint and starts the app through `/api/v2/hosted-app/start`. +FROM lambda-microvm-runner AS lambda-microvm-app-host + +ENV SANDBOX_HOSTED_APPS_ENABLED=true \ + SANDBOX_HOSTED_APP_PORT=3000 + +EXPOSE 3000/tcp + # ============================================================================ # Stage 3: Build the Rust launcher binary (Fedora for libkrun ABI) # ============================================================================ diff --git a/api/src/api/hosted-app.routes.test.ts b/api/src/api/hosted-app.routes.test.ts new file mode 100644 index 0000000..88e8c18 --- /dev/null +++ b/api/src/api/hosted-app.routes.test.ts @@ -0,0 +1,88 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test'; +import express from 'express'; +import type { Server } from 'node:http'; +import { config } from '../config'; +import { resetSessionWorkspaceStateForTests } from '../session-workspace'; +import v2Router from './v2'; + +let server: Server; +let baseUrl: string; +const savedHostedAppsEnabled = config.hosted_apps_enabled; +const savedSessionWorkspaceEnabled = config.session_workspace_enabled; + +beforeAll(async () => { + const app = express(); + app.use(express.urlencoded({ extended: true })); + app.use('/api/v2', v2Router); + await new Promise(resolve => { + server = app.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + baseUrl = `http://127.0.0.1:${typeof address === 'object' && address ? address.port : 0}`; +}); + +afterEach(() => { + config.hosted_apps_enabled = false; + config.session_workspace_enabled = false; + resetSessionWorkspaceStateForTests(); +}); + +afterAll(async () => { + await new Promise(resolve => server.close(() => resolve())); + config.hosted_apps_enabled = savedHostedAppsEnabled; + config.session_workspace_enabled = savedSessionWorkspaceEnabled; + resetSessionWorkspaceStateForTests(); +}); + +const post = (path: string, body: unknown = {}) => fetch(`${baseUrl}/api/v2${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), +}); + +describe('hosted-app route isolation', () => { + test('ordinary runner images do not expose hosted-app controls', async () => { + config.hosted_apps_enabled = false; + config.session_workspace_enabled = true; + + const response = await post('/hosted-app/start'); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ message: 'Not Found' }); + }); + + test('dedicated app hosts require the authenticated runtime-session binding', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await post('/hosted-app/start'); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ message: 'Missing runtime session header' }); + }); + + test('dedicated app hosts refuse ordinary execution requests', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await post('/execute', {}); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ message: 'Not Found' }); + }); + + test('status is session-bound and reports absence without starting a process', async () => { + config.hosted_apps_enabled = true; + config.session_workspace_enabled = true; + + const response = await fetch(`${baseUrl}/api/v2/hosted-app/status`, { + headers: { 'X-Runtime-Session-Id': 'rt_hosted_demo' }, + }); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + error: 'hosted_app_not_running', + message: 'No hosted app has been started', + }); + }); +}); diff --git a/api/src/api/lifecycle.ts b/api/src/api/lifecycle.ts index 28977ed..1963f7f 100644 --- a/api/src/api/lifecycle.ts +++ b/api/src/api/lifecycle.ts @@ -1,6 +1,7 @@ import express, { Router, type Request, type Response } from 'express'; import { logger } from '../logger'; import { bindSessionWorkspace, parseSessionBinding, unbindSessionWorkspace } from '../session-workspace'; +import { hostedAppSupervisor } from '../hosted-app'; /** * AWS Lambda MicroVM hook endpoints. The platform POSTs to @@ -97,7 +98,12 @@ lifecycleRouter.post('/suspend', ackHook('suspend')); lifecycleRouter.post('/terminate', (_req: Request, res: Response) => { logger.info({ hook: 'terminate' }, 'MicroVM lifecycle hook invoked'); - void unbindSessionWorkspace().catch((err) => logger.error({ err }, 'Failed to unbind session workspace on terminate')); + /* Stop before resetting the workspace so a resident process cannot race the + * recursive session cleanup. The platform does not need to wait for this + * best-effort cleanup before destroying the whole MicroVM. */ + void hostedAppSupervisor.shutdown() + .then(() => unbindSessionWorkspace()) + .catch((err) => logger.error({ err }, 'Failed to stop hosted app on terminate')); return res.status(200).json({ hook: 'terminate', status: 'ok' }); }); diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index b489573..0610469 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -32,6 +32,10 @@ import { pruneInputCache, storeCachedInputs, } from '../session-inputs'; +import { + HostedAppError, + hostedAppSupervisor, +} from '../hosted-app'; const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; @@ -388,6 +392,16 @@ router.use((req: Request, res: Response, next: NextFunction) => { next(); }); +/* A hosted-app image is a dedicated process host, not an execution runner. + * Keeping `/execute` structurally unavailable prevents a resident app and an + * NsJail job from sharing the pinned session UID/workspace concurrently. */ +router.use('/execute', (_req: Request, res: Response, next: NextFunction) => { + if (config.hosted_apps_enabled) { + return res.status(404).json({ message: 'Not Found' }); + } + next(); +}); + /** Replay PTC payloads (user code + tool definitions + inlined * `_ptc_history.json` + pyplot assets) can far exceed Express's default * ~100kb body limit. The parser is installed *here* rather than globally @@ -686,6 +700,75 @@ router.post('/session/restore', (req: Request, res: Response, next: NextFunction } return restoreSessionCheckpoint(req, res).catch(next); }); + +function requireHostedAppTarget( + _req: Request, + res: Response, + next: NextFunction, +): Response | void { + if (!config.hosted_apps_enabled) { + return res.status(404).json({ message: 'Not Found' }); + } + next(); +} + +function hostedAppFailure( + error: unknown, + res: Response, + next: NextFunction, +): Response | void { + if (error instanceof HostedAppError) { + return res.status(error.status).json({ error: error.code, message: error.message }); + } + next(error); +} + +/* Dedicated Lambda MicroVM resident-server adapter. The control plane first + * restores an immutable session checkpoint into this VM, then starts exactly + * one foreground process. Preview traffic uses a separate AWS token restricted + * to `hosted_app_port`; these control routes stay on the runner port. */ +router.post( + '/hosted-app/start', + requireHostedAppTarget, + express.json({ limit: '64kb' }), + (req: Request, res: Response, next: NextFunction) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + return hostedAppSupervisor.start(req.body) + .then(status => res.status(200).json(status)) + .catch(error => hostedAppFailure(error, res, next)); + }, +); + +router.get( + '/hosted-app/status', + requireHostedAppTarget, + (req: Request, res: Response) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + const status = hostedAppSupervisor.status(); + if (!status) { + return res.status(404).json({ + error: 'hosted_app_not_running', + message: 'No hosted app has been started', + }); + } + return res.status(200).json(status); + }, +); + +router.post( + '/hosted-app/stop', + requireHostedAppTarget, + express.json({ limit: '1kb' }), + (req: Request, res: Response, next: NextFunction) => { + const failure = bindSessionFromHeader(req); + if (failure) return res.status(failure.status).json(failure.body); + return hostedAppSupervisor.stop() + .then(status => status ? res.status(200).json(status) : res.status(204).send()) + .catch(next); + }, +); /** * Input delivery for backends whose sandbox cannot reach the file server. * diff --git a/api/src/config.ts b/api/src/config.ts index 7724ff0..a7ef4e9 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -71,6 +71,31 @@ export const config = { * session mode. An enabled runner additionally binds each request to a * workspace through the authenticated X-Runtime-Session-Id header. */ session_workspace_enabled: (process.env.SANDBOX_SESSION_WORKSPACE_ENABLED ?? 'false') === 'true', + /** + * Enables the Lambda-only hosted-app runner surface. This must only be set + * on a dedicated app-host MicroVM image: user application processes share + * that VM's network namespace and are therefore intentionally never started + * by the ordinary stateless/session execution runner. + */ + hosted_apps_enabled: (process.env.SANDBOX_HOSTED_APPS_ENABLED ?? 'false') === 'true', + hosted_app_port: safeInt(process.env.SANDBOX_HOSTED_APP_PORT, 3000), + hosted_app_start_timeout_ms: safeInt( + process.env.SANDBOX_HOSTED_APP_START_TIMEOUT_MS, + 30_000, + ), + hosted_app_stop_timeout_ms: safeInt( + process.env.SANDBOX_HOSTED_APP_STOP_TIMEOUT_MS, + 5_000, + ), + hosted_app_log_max_bytes: safeInt( + process.env.SANDBOX_HOSTED_APP_LOG_MAX_BYTES, + 64 * 1024, + ), + hosted_app_memory_max_bytes: safeInt( + process.env.SANDBOX_HOSTED_APP_MEMORY_MAX_BYTES, + 2 * 1024 * 1024 * 1024, + ), + hosted_app_pids_max: safeInt(process.env.SANDBOX_HOSTED_APP_PIDS_MAX, 128), job_uid_base: safeInt(process.env.SANDBOX_JOB_UID_BASE, 200000), job_gid_base: safeInt(process.env.SANDBOX_JOB_GID_BASE, 200000), job_uid_count: safeInt( diff --git a/api/src/hosted-app-launcher.sh b/api/src/hosted-app-launcher.sh new file mode 100644 index 0000000..ea866ed --- /dev/null +++ b/api/src/hosted-app-launcher.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -lt 5 ]; then + echo "usage: hosted-app-launcher [args...]" >&2 + exit 64 +fi + +CGROUP_PATH="$1" +APP_UID="$2" +APP_GID="$3" +shift 3 + +# This wrapper starts as root, joins the root-owned cgroup before any user code +# can fork, then irreversibly drops identity and capabilities. App descendants +# inherit the cgroup even if they daemonize or create a new process group. +printf '%s' "$$" > "${CGROUP_PATH}/cgroup.procs" +exec /usr/bin/setpriv \ + --no-new-privs \ + --reuid "$APP_UID" \ + --regid "$APP_GID" \ + --clear-groups \ + --inh-caps=-all \ + --ambient-caps=-all \ + --bounding-set=-all \ + -- "$@" diff --git a/api/src/hosted-app.test.ts b/api/src/hosted-app.test.ts new file mode 100644 index 0000000..beb9a22 --- /dev/null +++ b/api/src/hosted-app.test.ts @@ -0,0 +1,324 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { PassThrough } from 'node:stream'; +import type { SpawnOptions } from 'node:child_process'; +import type { Runtime } from './runtime'; +import type { SessionWorkspace } from './session-workspace'; +import { + HostedAppError, + HostedAppSupervisor, + type HostedAppDependencies, + type HostedAppStartRequest, +} from './hosted-app'; +import { config } from './config'; + +interface FakeChild extends EventEmitter { + pid: number; + stdout: PassThrough; + stderr: PassThrough; +} + +const savedConfig = { + port: config.hosted_app_port, + start: config.hosted_app_start_timeout_ms, + stop: config.hosted_app_stop_timeout_ms, + logs: config.hosted_app_log_max_bytes, +}; + +let roots: string[] = []; + +beforeEach(() => { + config.hosted_app_port = 3123; + config.hosted_app_start_timeout_ms = 25; + config.hosted_app_stop_timeout_ms = 25; + config.hosted_app_log_max_bytes = 64; +}); + +afterEach(async () => { + config.hosted_app_port = savedConfig.port; + config.hosted_app_start_timeout_ms = savedConfig.start; + config.hosted_app_stop_timeout_ms = savedConfig.stop; + config.hosted_app_log_max_bytes = savedConfig.logs; + await Promise.all(roots.map(root => fsp.rm(root, { recursive: true, force: true }))); + roots = []; +}); + +async function workspace(): Promise { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-')); + roots.push(root); + await fsp.writeFile(path.join(root, 'server.js'), 'serve();'); + await fsp.mkdir(path.join(root, 'app')); + return root; +} + +function fakeRuntime(): Runtime { + return { + language: 'node', + version: { raw: '22.0.0' } as Runtime['version'], + aliases: [], + pkgdir: '/pkgs/node/22', + compiled: false, + env_vars: { PATH: '/pkgs/node/22/bin:/usr/bin' }, + timeouts: { compile: 0, run: 0 }, + cpu_times: { compile: 0, run: 0 }, + memory_limits: { compile: 0, run: 0 }, + max_process_count: 64, + max_open_files: 2048, + max_file_size: 10_000_000, + output_max_size: 1024, + }; +} + +function fakeChild(pid = 4242): FakeChild { + const child = new EventEmitter() as FakeChild; + child.pid = pid; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + return child; +} + +function request(overrides: Partial = {}): HostedAppStartRequest { + return { + app_id: 'demo', + revision: 'rev-1', + language: 'node', + version: '>=22', + entrypoint: 'server.js', + ...overrides, + }; +} + +function dependencies( + root: string, + options: { + probe?: boolean; + guardError?: Error; + killCgroup?: () => Promise; + } = {}, +): { + deps: HostedAppDependencies; + spawns: Array<{ command: string; args: readonly string[]; options: SpawnOptions }>; + guards: number[]; + cgroupKills: string[]; + kills: Array<{ pid: number; signal: NodeJS.Signals }>; + children: FakeChild[]; +} { + const spawns: Array<{ command: string; args: readonly string[]; options: SpawnOptions }> = []; + const guards: number[] = []; + const cgroupKills: string[] = []; + const kills: Array<{ pid: number; signal: NodeJS.Signals }> = []; + const children: FakeChild[] = []; + const getSession = () => ({ + ownership: async () => ({ dir: root, uid: 200123, gid: 200123 }), + } as SessionWorkspace); + const deps: HostedAppDependencies = { + getSession, + resolveRuntime: () => fakeRuntime(), + spawnApp: ((command: string, args: readonly string[], spawnOptions: SpawnOptions) => { + spawns.push({ command, args, options: spawnOptions }); + const child = fakeChild(4242 + children.length); + children.push(child); + return child as unknown as ReturnType; + }) as HostedAppDependencies['spawnApp'], + prepareCgroup: async () => {}, + killCgroup: async () => { + cgroupKills.push('kill'); + await options.killCgroup?.(); + }, + installNetworkGuard: async uid => { + guards.push(uid); + if (options.guardError) throw options.guardError; + }, + probePort: async () => options.probe ?? true, + killProcessGroup: (pid, signal) => { + kills.push({ pid, signal }); + const child = children.find(candidate => candidate.pid === pid); + queueMicrotask(() => child?.emit('exit', null, signal)); + }, + now: () => new Date('2026-08-21T12:00:00.000Z'), + }; + return { deps, spawns, guards, cgroupKills, kills, children }; +} + +describe('HostedAppSupervisor', () => { + test('starts a runtime as the session UID with a curated fixed-port environment', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const status = await supervisor.start(request({ + args: ['--production'], + env: { + APP_NAME: 'demo', + HOME: '/attacker', + port: '9999', + HOST: 'attacker.invalid', + LD_PRELOAD: '/tmp/evil.so', + }, + })); + + expect(status).toMatchObject({ + app_id: 'demo', + revision: 'rev-1', + state: 'running', + port: 3123, + pid: 4242, + }); + expect(fixture.guards).toEqual([200123]); + expect(fixture.spawns).toHaveLength(1); + const launch = fixture.spawns[0]; + expect(launch.command).toBe('/usr/local/bin/codeapi-hosted-app-launcher'); + const realRoot = await fsp.realpath(root); + expect(launch.args).toEqual([ + '/sys/fs/cgroup/codeapi_hosted_app', + '200123', + '200123', + '/bin/bash', + '/pkgs/node/22/run', + path.join(realRoot, 'server.js'), + '--production', + ]); + expect(launch.options).toMatchObject({ + cwd: realRoot, + detached: true, + }); + expect(launch.options.env).toMatchObject({ + APP_NAME: 'demo', + HOME: root, + HOST: '0.0.0.0', + PORT: '3123', + PATH: '/pkgs/node/22/bin:/usr/bin', + }); + expect(launch.options.env).not.toHaveProperty('port'); + expect(launch.options.env).not.toHaveProperty('LD_PRELOAD'); + await supervisor.shutdown(); + }); + + test('is idempotent for the exact same immutable revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + const spec = request({ env: { B: '2', A: '1' } }); + + const first = await supervisor.start(spec); + const second = await supervisor.start(request({ env: { A: '1', B: '2' } })); + + expect(second).toEqual(first); + expect(fixture.spawns).toHaveLength(1); + await supervisor.shutdown(); + }); + + test('rejects changed launch settings under an existing revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const error = await supervisor.start(request({ args: ['changed'] })).catch(value => value); + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_revision_conflict'); + expect(fixture.spawns).toHaveLength(1); + await supervisor.shutdown(); + }); + + test('stops the old process group before launching a new revision', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + const status = await supervisor.start(request({ revision: 'rev-2' })); + + expect(status.revision).toBe('rev-2'); + expect(fixture.kills).toEqual([{ pid: 4242, signal: 'SIGTERM' }]); + expect(fixture.cgroupKills.length).toBeGreaterThan(0); + expect(fixture.spawns).toHaveLength(2); + await supervisor.shutdown(); + }); + + test('fails closed before spawning when the network guard cannot be installed', async () => { + const root = await workspace(); + const fixture = dependencies(root, { guardError: new Error('iptables unavailable') }); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const error = await supervisor.start(request()).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_isolation_failed'); + expect(fixture.spawns).toHaveLength(0); + }); + + test('rejects symlink entrypoints even when the target is a regular file', async () => { + const root = await workspace(); + const outside = await fsp.mkdtemp(path.join(os.tmpdir(), 'hosted-app-outside-')); + roots.push(outside); + await fsp.writeFile(path.join(outside, 'outside.js'), 'steal();'); + await fsp.symlink(path.join(outside, 'outside.js'), path.join(root, 'linked.js')); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + + const error = await supervisor.start(request({ entrypoint: 'linked.js' })).catch(value => value); + + expect(error).toBeInstanceOf(HostedAppError); + expect(error.code).toBe('hosted_app_path_escape'); + expect(fixture.guards).toHaveLength(0); + expect(fixture.spawns).toHaveLength(0); + }); + + test('retains only the bounded tail of process logs', async () => { + const root = await workspace(); + config.hosted_app_log_max_bytes = 8; + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].stdout.write('0123456789'); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(supervisor.status()?.stdout).toBe('23456789'); + await supervisor.shutdown(); + }); + + test('reaps the cgroup when the tracked parent exits unexpectedly', async () => { + const root = await workspace(); + const fixture = dependencies(root); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].emit('exit', 1, null); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(supervisor.status()?.state).toBe('failed'); + expect(fixture.cgroupKills.length).toBeGreaterThan(0); + await supervisor.shutdown(); + }); + + test('waits for unexpected-exit cleanup before launching a replacement revision', async () => { + const root = await workspace(); + let releaseCleanup!: () => void; + const cleanupBlocked = new Promise(resolve => { releaseCleanup = resolve; }); + let cleanupCalls = 0; + const fixture = dependencies(root, { + killCgroup: async () => { + cleanupCalls += 1; + if (cleanupCalls === 1) await cleanupBlocked; + }, + }); + const supervisor = new HostedAppSupervisor(fixture.deps); + await supervisor.start(request()); + + fixture.children[0].emit('exit', 1, null); + await Promise.resolve(); + const replacement = supervisor.start(request({ revision: 'rev-2' })); + await Promise.resolve(); + expect(fixture.spawns).toHaveLength(1); + + releaseCleanup(); + expect((await replacement).revision).toBe('rev-2'); + expect(fixture.spawns).toHaveLength(2); + await supervisor.shutdown(); + }); +}); diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts new file mode 100644 index 0000000..a583b52 --- /dev/null +++ b/api/src/hosted-app.ts @@ -0,0 +1,666 @@ +import { execFile, spawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import * as fsp from 'node:fs/promises'; +import * as net from 'node:net'; +import * as path from 'node:path'; +import type { Readable } from 'node:stream'; +import { promisify } from 'node:util'; +import { config } from './config'; +import { filterExtraEnvVars } from './job'; +import { logger } from './logger'; +import { getLatestRuntimeMatchingLanguageVersion } from './runtime'; +import { getBoundSessionWorkspace, type SessionWorkspace } from './session-workspace'; +import { ValidationError, validateFilePath } from './validation'; + +const execFileAsync = promisify(execFile); +const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const HOSTED_APP_EGRESS_CHAIN = 'CODEAPI_HOSTED_APP_EGRESS'; +/* Sibling of sandbox_api, not its child: the API process lives in sandbox_api + * and cgroup v2 forbids enabling domain controllers below a populated parent. */ +const HOSTED_APP_CGROUP = '/sys/fs/cgroup/codeapi_hosted_app'; +const HOSTED_APP_LAUNCHER = '/usr/local/bin/codeapi-hosted-app-launcher'; +const MAX_ARGS = 64; +const MAX_ARG_BYTES = 4096; +const MAX_ENV_VARS = 64; +const MAX_ENV_VALUE_BYTES = 4096; +const MAX_ENV_BYTES = 32 * 1024; +const PROBE_INTERVAL_MS = 100; + +export interface HostedAppStartRequest { + app_id: string; + revision: string; + language: string; + version: string; + entrypoint: string; + cwd?: string; + args?: string[]; + env?: Record; +} + +interface NormalizedHostedAppRequest extends HostedAppStartRequest { + cwd: string; + args: string[]; + env: Record; +} + +export type HostedAppState = 'starting' | 'running' | 'stopping' | 'stopped' | 'failed'; + +export interface HostedAppStatus { + app_id: string; + revision: string; + state: HostedAppState; + port: number; + pid?: number; + started_at: string; + exited_at?: string; + exit_code?: number; + signal?: NodeJS.Signals; + message?: string; + stdout: string; + stderr: string; +} + +export class HostedAppError extends Error { + constructor( + readonly code: string, + message: string, + readonly status: number, + ) { + super(message); + this.name = 'HostedAppError'; + } +} + +interface ActiveHostedApp { + request: NormalizedHostedAppRequest; + specKey: string; + process?: HostedAppChild; + status: HostedAppStatus; +} + +interface HostedAppChild extends ChildProcess { + readonly pid: number; + readonly stdout: Readable; + readonly stderr: Readable; +} + +type SpawnApp = (command: string, args: readonly string[], options: SpawnOptions) => HostedAppChild; +type ResolveRuntime = typeof getLatestRuntimeMatchingLanguageVersion; + +export interface HostedAppDependencies { + getSession: () => SessionWorkspace | undefined; + resolveRuntime: ResolveRuntime; + spawnApp: SpawnApp; + prepareCgroup: () => Promise; + killCgroup: () => Promise; + installNetworkGuard: (uid: number) => Promise; + probePort: (port: number) => Promise; + killProcessGroup: (pid: number, signal: NodeJS.Signals) => void; + now: () => Date; +} + +function byteLength(value: string): number { + return Buffer.byteLength(value, 'utf8'); +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const proto = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +function normalizeHostedAppRequest(value: unknown): NormalizedHostedAppRequest { + if (!isPlainObject(value)) { + throw new HostedAppError('invalid_hosted_app_request', 'request body must be an object', 400); + } + + const stringField = (name: string): string => { + const field = value[name]; + if (typeof field !== 'string' || field.length === 0) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `${name} must be a non-empty string`, + 400, + ); + } + if (field.includes('\0')) { + throw new HostedAppError('invalid_hosted_app_request', `${name} must not contain NUL`, 400); + } + return field; + }; + + const appId = stringField('app_id'); + const revision = stringField('revision'); + const language = stringField('language'); + const version = stringField('version'); + const entrypoint = stringField('entrypoint'); + if (!APP_ID_PATTERN.test(appId)) { + throw new HostedAppError('invalid_hosted_app_request', 'app_id is malformed', 400); + } + if (!REVISION_PATTERN.test(revision)) { + throw new HostedAppError('invalid_hosted_app_request', 'revision is malformed', 400); + } + try { + validateFilePath(entrypoint, '/tmp/codeapi-hosted-app-validation'); + } catch (error) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `entrypoint is invalid: ${error instanceof Error ? error.message : 'invalid path'}`, + 400, + ); + } + + const cwdValue = value.cwd ?? '.'; + if (typeof cwdValue !== 'string' || cwdValue.includes('\0')) { + throw new HostedAppError('invalid_hosted_app_request', 'cwd must be a string', 400); + } + if (cwdValue !== '.') { + try { + /* validateFilePath is also the canonical relative-path validator. A cwd + * is allowed to name a directory; append a sentinel so trailing-slash + * and directory-root cases retain the same traversal checks. */ + validateFilePath(path.posix.join(cwdValue, '.codeapi-cwd'), '/tmp/codeapi-hosted-app-validation'); + if (path.posix.normalize(cwdValue) !== cwdValue || cwdValue.endsWith('/')) { + throw new ValidationError('cwd must be a canonical relative path'); + } + } catch (error) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `cwd is invalid: ${error instanceof Error ? error.message : 'invalid path'}`, + 400, + ); + } + } + + const argsValue = value.args ?? []; + if ( + !Array.isArray(argsValue) + || argsValue.length > MAX_ARGS + || argsValue.some(arg => ( + typeof arg !== 'string' + || arg.includes('\0') + || byteLength(arg) > MAX_ARG_BYTES + )) + ) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `args must contain at most ${MAX_ARGS} bounded strings`, + 400, + ); + } + + const envValue = value.env ?? {}; + if (!isPlainObject(envValue) || Object.keys(envValue).length > MAX_ENV_VARS) { + throw new HostedAppError( + 'invalid_hosted_app_request', + `env must be an object with at most ${MAX_ENV_VARS} entries`, + 400, + ); + } + const env: Record = {}; + let envBytes = 0; + for (const [key, raw] of Object.entries(envValue)) { + if ( + !ENV_NAME_PATTERN.test(key) + || typeof raw !== 'string' + || raw.includes('\0') + || byteLength(raw) > MAX_ENV_VALUE_BYTES + ) { + throw new HostedAppError('invalid_hosted_app_request', `env.${key} is invalid`, 400); + } + envBytes += byteLength(key) + byteLength(raw); + if (envBytes > MAX_ENV_BYTES) { + throw new HostedAppError('invalid_hosted_app_request', 'env is too large', 400); + } + env[key] = raw; + } + + return { + app_id: appId, + revision, + language, + version, + entrypoint, + cwd: cwdValue, + args: [...argsValue] as string[], + env, + }; +} + +function canonicalSpecKey(request: NormalizedHostedAppRequest): string { + return JSON.stringify({ + ...request, + env: Object.fromEntries(Object.entries(request.env).sort(([a], [b]) => a.localeCompare(b))), + }); +} + +function isInside(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`)); +} + +async function resolveWorkspacePaths( + workspaceDir: string, + request: NormalizedHostedAppRequest, +): Promise<{ cwd: string; entrypoint: string }> { + const realRoot = await fsp.realpath(workspaceDir); + const cwd = await fsp.realpath(path.resolve(realRoot, request.cwd)).catch(() => { + throw new HostedAppError('hosted_app_cwd_missing', 'cwd does not exist', 400); + }); + if (!isInside(realRoot, cwd)) { + throw new HostedAppError('hosted_app_path_escape', 'cwd escapes the session workspace', 400); + } + const cwdStat = await fsp.stat(cwd); + if (!cwdStat.isDirectory()) { + throw new HostedAppError('hosted_app_cwd_missing', 'cwd is not a directory', 400); + } + + const requestedEntrypoint = path.resolve(realRoot, request.entrypoint); + const entrypointLstat = await fsp.lstat(requestedEntrypoint).catch(() => { + throw new HostedAppError('hosted_app_entrypoint_missing', 'entrypoint does not exist', 400); + }); + if (entrypointLstat.isSymbolicLink()) { + throw new HostedAppError('hosted_app_path_escape', 'entrypoint must not be a symbolic link', 400); + } + const entrypoint = await fsp.realpath(requestedEntrypoint); + if (!isInside(realRoot, entrypoint)) { + throw new HostedAppError('hosted_app_path_escape', 'entrypoint escapes the session workspace', 400); + } + const entrypointStat = await fsp.stat(entrypoint); + if (!entrypointStat.isFile()) { + throw new HostedAppError('hosted_app_entrypoint_missing', 'entrypoint is not a file', 400); + } + return { cwd, entrypoint }; +} + +function appendBounded(current: string, chunk: Buffer | string): string { + const next = current + chunk.toString(); + const bytes = Buffer.byteLength(next); + if (bytes <= config.hosted_app_log_max_bytes) return next; + return Buffer.from(next).subarray(bytes - config.hosted_app_log_max_bytes).toString(); +} + +async function commandSucceeds(binary: string, args: string[]): Promise { + try { + await execFileAsync(binary, args); + return true; + } catch { + return false; + } +} + +/** + * Hosted apps receive inbound preview traffic, but may not initiate network + * connections. Besides blocking internet egress, this prevents the untrusted + * app UID from calling the root-owned control listener on localhost. Reply + * packets for accepted inbound connections remain allowed by conntrack. + */ +export async function installHostedAppNetworkGuard(uid: number): Promise { + for (const binary of ['/usr/sbin/iptables', '/usr/sbin/ip6tables']) { + if (!(await commandSucceeds(binary, ['-w', '5', '-L', HOSTED_APP_EGRESS_CHAIN]))) { + await execFileAsync(binary, ['-w', '5', '-N', HOSTED_APP_EGRESS_CHAIN]); + } + await execFileAsync(binary, ['-w', '5', '-F', HOSTED_APP_EGRESS_CHAIN]); + await execFileAsync(binary, [ + '-w', '5', '-A', HOSTED_APP_EGRESS_CHAIN, + '-m', 'conntrack', '--ctstate', 'ESTABLISHED,RELATED', '-j', 'ACCEPT', + ]); + await execFileAsync(binary, ['-w', '5', '-A', HOSTED_APP_EGRESS_CHAIN, '-j', 'REJECT']); + const jump = [ + '-m', 'owner', '--uid-owner', String(uid), '-j', HOSTED_APP_EGRESS_CHAIN, + ]; + if (!(await commandSucceeds(binary, ['-w', '5', '-C', 'OUTPUT', ...jump]))) { + await execFileAsync(binary, ['-w', '5', '-I', 'OUTPUT', '1', ...jump]); + } + } +} + +/** Create a process-tree boundary owned only by the root runner. The launcher + * moves itself here before dropping to the session UID, so every descendant + * inherits the cgroup and cannot escape it by daemonizing or calling setsid. */ +export async function prepareHostedAppCgroup(): Promise { + await fsp.mkdir(HOSTED_APP_CGROUP, { recursive: true }); + /* cgroup.kill (Linux 5.14+) is required, not an optional optimization: it is + * the primitive that prevents a setsid()/double-fork descendant escaping + * revision replacement. Fail closed on kernels that do not expose it. */ + await fsp.access(path.join(HOSTED_APP_CGROUP, 'cgroup.kill')); + await killHostedAppCgroup(); + for (let attempt = 0; attempt < 20; attempt += 1) { + const events = await fsp.readFile(path.join(HOSTED_APP_CGROUP, 'cgroup.events'), 'utf8'); + if (/^populated 0$/m.test(events)) break; + if (attempt === 19) throw new Error('hosted-app cgroup did not become empty'); + await new Promise(resolve => setTimeout(resolve, 25)); + } + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'memory.max'), String( + config.hosted_app_memory_max_bytes, + )); + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'pids.max'), String( + config.hosted_app_pids_max, + )); +} + +/** `cgroup.kill` reaches descendants that changed session/process group. */ +export async function killHostedAppCgroup(): Promise { + try { + await fsp.writeFile(path.join(HOSTED_APP_CGROUP, 'cgroup.kill'), '1'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } +} + +export function probeHostedAppPort(port: number): Promise { + return new Promise(resolve => { + const socket = net.createConnection({ host: '127.0.0.1', port }); + socket.setTimeout(500); + socket.once('connect', () => { + socket.destroy(); + resolve(true); + }); + const fail = (): void => { + socket.destroy(); + resolve(false); + }; + socket.once('error', fail); + socket.once('timeout', fail); + }); +} + +function killHostedAppProcessGroup(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(-pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error; + } +} + +function publicStatus(active: ActiveHostedApp): HostedAppStatus { + return { ...active.status }; +} + +export class HostedAppSupervisor { + private active: ActiveHostedApp | undefined; + private transition: Promise = Promise.resolve(); + + constructor(private readonly deps: HostedAppDependencies = { + getSession: getBoundSessionWorkspace, + resolveRuntime: getLatestRuntimeMatchingLanguageVersion, + spawnApp: (command, args, options) => spawn(command, args, options) as unknown as HostedAppChild, + prepareCgroup: prepareHostedAppCgroup, + killCgroup: killHostedAppCgroup, + installNetworkGuard: installHostedAppNetworkGuard, + probePort: probeHostedAppPort, + killProcessGroup: killHostedAppProcessGroup, + now: () => new Date(), + }) {} + + status(): HostedAppStatus | undefined { + return this.active ? publicStatus(this.active) : undefined; + } + + async start(rawRequest: unknown): Promise { + return this.serialize(() => this.startImpl(rawRequest)); + } + + async stop(): Promise { + return this.serialize(() => this.stopImpl()); + } + + async shutdown(): Promise { + await this.stop(); + } + + private async serialize(operation: () => Promise): Promise { + const previous = this.transition; + let release!: () => void; + this.transition = new Promise(resolve => { release = resolve; }); + await previous; + try { + return await operation(); + } finally { + release(); + } + } + + private async startImpl(rawRequest: unknown): Promise { + const request = normalizeHostedAppRequest(rawRequest); + const specKey = canonicalSpecKey(request); + if (this.active?.status.state === 'running' && this.active.specKey === specKey) { + return publicStatus(this.active); + } + if ( + this.active + && this.active.request.app_id === request.app_id + && this.active.request.revision === request.revision + && this.active.specKey !== specKey + ) { + throw new HostedAppError( + 'hosted_app_revision_conflict', + 'an app revision is immutable; use a new revision for changed launch settings', + 409, + ); + } + await this.stopImpl(); + + const session = this.deps.getSession(); + if (!session) { + throw new HostedAppError( + 'hosted_app_session_required', + 'a bound stateful runtime session is required', + 409, + ); + } + const runtime = this.deps.resolveRuntime(request.language, request.version); + if (!runtime) { + throw new HostedAppError( + 'hosted_app_runtime_not_found', + `runtime ${request.language}@${request.version} is not installed`, + 400, + ); + } + if (runtime.compiled) { + throw new HostedAppError( + 'hosted_app_runtime_unsupported', + 'compiled runtimes are not supported by the resident-server adapter', + 400, + ); + } + + const ownership = await session.ownership(); + const workspace = await resolveWorkspacePaths(ownership.dir, request); + try { + await this.deps.prepareCgroup(); + await this.deps.installNetworkGuard(ownership.uid); + } catch (error) { + logger.error({ err: error, uid: ownership.uid }, 'Hosted-app isolation setup failed'); + throw new HostedAppError( + 'hosted_app_isolation_failed', + 'hosted app could not be started safely', + 503, + ); + } + + const status: HostedAppStatus = { + app_id: request.app_id, + revision: request.revision, + state: 'starting', + port: config.hosted_app_port, + started_at: this.deps.now().toISOString(), + stdout: '', + stderr: '', + }; + const active: ActiveHostedApp = { request, specKey, status }; + this.active = active; + + const callerEnv = filterExtraEnvVars(request.env); + for (const key of Object.keys(callerEnv)) { + if (key.toUpperCase() === 'PORT' || key.toUpperCase() === 'HOST') { + delete callerEnv[key]; + } + } + const env: NodeJS.ProcessEnv = { + ...callerEnv, + ...runtime.env_vars, + HOME: ownership.dir, + HOST: '0.0.0.0', + PORT: String(config.hosted_app_port), + SANDBOX_LANGUAGE: runtime.language, + }; + const command = HOSTED_APP_LAUNCHER; + const args = [ + HOSTED_APP_CGROUP, + String(ownership.uid), + String(ownership.gid), + '/bin/bash', + path.join(runtime.pkgdir, 'run'), + workspace.entrypoint, + ...request.args, + ]; + + let child: HostedAppChild; + try { + child = this.deps.spawnApp(command, args, { + cwd: workspace.cwd, + env, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (error) { + active.status.state = 'failed'; + active.status.exited_at = this.deps.now().toISOString(); + active.status.message = 'hosted app process could not be spawned'; + throw new HostedAppError('hosted_app_spawn_failed', active.status.message, 500); + } + active.process = child; + active.status.pid = child.pid; + child.stdout.on('data', chunk => { + active.status.stdout = appendBounded(active.status.stdout, chunk); + }); + child.stderr.on('data', chunk => { + active.status.stderr = appendBounded(active.status.stderr, chunk); + }); + child.once('error', error => { + active.status.state = 'failed'; + active.status.exited_at = this.deps.now().toISOString(); + active.status.message = error.message; + }); + child.once('exit', (code, signal) => { + active.process = undefined; + active.status.exited_at = this.deps.now().toISOString(); + if (code !== null) active.status.exit_code = code; + if (signal !== null) active.status.signal = signal; + if (active.status.state === 'stopping') { + active.status.state = 'stopped'; + } else if (active.status.state !== 'stopped') { + active.status.state = 'failed'; + active.status.message ??= 'hosted app exited'; + } + /* A daemonized descendant can outlive the tracked launcher process and + * process group. Queue unexpected cleanup in the same transition chain + * as start/stop: a detached sweep must never land after a new revision + * has entered this shared cgroup. */ + if (active.status.state !== 'stopped') { + void this.serialize(async () => { + if (active.status.state === 'stopped') return; + await this.deps.killCgroup(); + }).catch(error => { + logger.error( + { err: error, appId: active.request.app_id }, + 'Hosted-app cgroup cleanup failed', + ); + }); + } + }); + + const deadline = Date.now() + config.hosted_app_start_timeout_ms; + while (Date.now() < deadline) { + if (active.status.state === 'failed') { + await this.stopImpl(true); + active.status.state = 'failed'; + throw new HostedAppError( + 'hosted_app_start_failed', + active.status.message ?? 'hosted app exited before becoming ready', + 502, + ); + } + if (await this.deps.probePort(config.hosted_app_port)) { + active.status.state = 'running'; + logger.info( + { appId: request.app_id, revision: request.revision, pid: child.pid }, + 'Hosted app started', + ); + return publicStatus(active); + } + await new Promise(resolve => setTimeout(resolve, PROBE_INTERVAL_MS)); + } + + active.status.message = `hosted app did not listen on port ${config.hosted_app_port}`; + await this.stopImpl(true); + active.status.state = 'failed'; + throw new HostedAppError('hosted_app_start_timeout', active.status.message, 504); + } + + private async stopImpl(preserveActive = false): Promise { + const active = this.active; + if (!active) return undefined; + const child = active.process; + if (!child?.pid) { + await this.deps.killCgroup(); + if (!preserveActive) this.active = undefined; + return publicStatus(active); + } + + active.status.state = 'stopping'; + this.deps.killProcessGroup(child.pid, 'SIGTERM'); + const exited = new Promise(resolve => child.once('exit', () => resolve(true))); + let timer: ReturnType | undefined; + const timedOut = new Promise(resolve => { + timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms); + timer.unref?.(); + }); + const stopped = await Promise.race([exited, timedOut]); + if (timer) clearTimeout(timer); + if (!stopped && active.process?.pid) { + await this.deps.killCgroup(); + await Promise.race([ + new Promise(resolve => child.once('exit', () => resolve())), + new Promise(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)), + ]); + } + /* Always sweep the cgroup: the tracked parent may have exited cleanly + * while a daemonized descendant stayed alive in a different process group. */ + await this.deps.killCgroup(); + active.status.state = 'stopped'; + active.status.exited_at ??= this.deps.now().toISOString(); + const status = publicStatus(active); + if (!preserveActive) this.active = undefined; + return status; + } +} + +export function validateHostedAppStartup(): void { + if (!config.hosted_apps_enabled) return; + const bindParts = config.bind_address.split(':'); + const runnerPort = Number(bindParts[bindParts.length - 1]); + const failures: string[] = []; + if (!config.session_workspace_enabled) { + failures.push('SANDBOX_SESSION_WORKSPACE_ENABLED must be true'); + } + if (config.hosted_app_port < 1024 || config.hosted_app_port > 65535) { + failures.push('SANDBOX_HOSTED_APP_PORT must be between 1024 and 65535'); + } + if (config.hosted_app_port === runnerPort) { + failures.push('SANDBOX_HOSTED_APP_PORT must differ from PORT'); + } + if (!config.use_cgroupv2) { + failures.push('SANDBOX_USE_CGROUPV2 must be true'); + } + if (process.getuid?.() !== 0) { + failures.push('the dedicated hosted-app runner must start as root'); + } + if (failures.length > 0) { + throw new Error(`Invalid hosted-app runner configuration: ${failures.join('; ')}`); + } +} + +export const hostedAppSupervisor = new HostedAppSupervisor(); diff --git a/api/src/index.ts b/api/src/index.ts index 629436e..cfbc936 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -8,6 +8,7 @@ import { httpMetricsMiddleware, metricsHandler } from './metrics'; import { positiveInt, shutdownTelemetry, traceHttpRequest } from './telemetry'; import { startWarmupCommand } from './warmup'; import { stopToolCallSocketProxy } from './tool-call-socket-process'; +import { hostedAppSupervisor, validateHostedAppStartup } from './hosted-app'; import v2Router from './api/v2'; import lifecycleRouter, { LIFECYCLE_HOOK_BASE_PATH } from './api/lifecycle'; @@ -66,6 +67,7 @@ app.use((err: HttpError, _req: express.Request, res: express.Response, _next: ex }); async function main(): Promise { + validateHostedAppStartup(); validateHardenedSandboxStartup(); await initializeSandboxWorkspaceIsolation(); await startWarmupCommand(); @@ -111,6 +113,9 @@ async function main(): Promise { shuttingDown = true; stopWorkspaceReaper(); await closeHttpServerWithTimeout(); + await hostedAppSupervisor.shutdown().catch((err) => { + logger.warn({ err }, 'Hosted-app process shutdown failed'); + }); await stopToolCallSocketProxy().catch((err) => { logger.warn({ err }, 'Tool-call socket proxy shutdown failed'); }); diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 309d6ec..2283bbd 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -142,6 +142,64 @@ IMAGE_DIGEST=sha256:<64-hex-digest> \ scripts/build-lambda-microvm-artifact.sh zip upload ``` +### Dedicated hosted-app image (experimental) + +Resident web servers use a separate `lambda-microvm-app-host` image. They do +not run as background children of `/execute`: an execution's PID/network +namespaces end with that execution, while the app host intentionally keeps one +supervised foreground process alive for the MicroVM lease. Build its immutable +artifact through the same provenance-checked pipeline: + +```bash +MICROVM_IMAGE_TARGET=lambda-microvm-app-host \ + ECR_URI="$ECR_URI" S3_URI="$S3_URI" IMAGE_TAG="$IMAGE_TAG" \ + scripts/build-lambda-microvm-artifact.sh build push zip upload +# → app-host tags/artifacts are distinct from the normal runner +``` + +The app-host contract is deliberately narrow: + +1. Launch a MicroVM from the dedicated app-host image and wait for port 8080. +2. Mint a control token restricted to port 8080. +3. Restore an immutable stateful-session checkpoint with + `X-Runtime-Session-Id`, exactly as for a replacement session runner. +4. `POST /api/v2/hosted-app/start` on port 8080 with the same session header: + + ```json + { + "app_id": "my-app", + "revision": "rev-1", + "language": "node", + "version": ">=22", + "entrypoint": "server.js", + "cwd": ".", + "args": [], + "env": {} + } + ``` + + The process must listen on `HOST=0.0.0.0` and the injected `PORT` (3000 by + default). Start is idempotent for an identical revision; changed launch + settings require a new revision. +5. Mint a separate preview token restricted to port 3000. Keep the endpoint and + both AWS credentials behind a CodeAPI preview gateway; never put a raw AWS + proxy token in browser-visible HTML or JavaScript. + +Hosted mode disables ordinary `/api/v2/execute`. The app runs as the +session-workspace UID with a curated environment, and the runner installs +fail-closed IPv4 and IPv6 OUTPUT rules before spawn: responses to inbound +preview traffic are allowed, but new outbound connections (including calls to +the root-owned control listener on localhost) are rejected. One app runs per +MicroVM. A restored checkpoint is a revision copy, not a live shared filesystem +with the coding VM. + +Lambda suspend/resume preserves the resident process, but the eight-hour hard +lifetime does not. The higher-level hosted-app control plane must therefore +retain the immutable revision/checkpoint identity, relaunch, restore, and start +again after expiry. Static assets and request-shaped handlers should remain on +cheaper stateless delivery paths; this target is only the resident-server +adapter. + ### 3. Generate the split execution-manifest keys The worker signs each execution manifest; the runner only receives the public diff --git a/scripts/build-lambda-microvm-artifact.sh b/scripts/build-lambda-microvm-artifact.sh index 8e8869b..6ad13b7 100755 --- a/scripts/build-lambda-microvm-artifact.sh +++ b/scripts/build-lambda-microvm-artifact.sh @@ -7,7 +7,7 @@ # in a same-account ECR repo (Lambda's build infra can pull it there). # # Stages (each optional, in order): -# build docker buildx the arm64 lambda-microvm-runner target (no AWS) +# build docker buildx the selected arm64 MicroVM target (no AWS) # push push to ECR (needs AWS_PROFILE + repo) # zip render the code-artifact Dockerfile and zip it (no AWS) # upload upload the zip to S3 (needs AWS_PROFILE + bucket) @@ -24,6 +24,8 @@ # S3_URI e.g. s3://codeapi-microvm-artifacts/runner # AWS_PROFILE e.g. librechat-dev # AWS_REGION required for push/upload +# MICROVM_IMAGE_TARGET lambda-microvm-runner (default) or +# lambda-microvm-app-host set -euo pipefail cd "$(dirname "$0")/.." @@ -37,8 +39,25 @@ if [ -z "${IMAGE_TAG:-}" ]; then fi ECR_URI="${ECR_URI:-}" S3_URI="${S3_URI:-}" -OUT_DIR="${OUT_DIR:-.build-lambda-microvm}" -LOCAL_TAG="codeapi-lambda-microvm-runner:${IMAGE_TAG}" +MICROVM_IMAGE_TARGET="${MICROVM_IMAGE_TARGET:-lambda-microvm-runner}" +case "$MICROVM_IMAGE_TARGET" in + lambda-microvm-runner) + ARTIFACT_KIND="runner" + PUBLISHED_TAG="$IMAGE_TAG" + DEFAULT_OUT_DIR=".build-lambda-microvm" + ;; + lambda-microvm-app-host) + ARTIFACT_KIND="app-host" + PUBLISHED_TAG="app-host-${IMAGE_TAG}" + DEFAULT_OUT_DIR=".build-lambda-microvm-app-host" + ;; + *) + echo "MICROVM_IMAGE_TARGET must be lambda-microvm-runner or lambda-microvm-app-host" >&2 + exit 1 + ;; +esac +OUT_DIR="${OUT_DIR:-$DEFAULT_OUT_DIR}" +LOCAL_TAG="codeapi-${MICROVM_IMAGE_TARGET}:${IMAGE_TAG}" IMAGE_DIGEST="${IMAGE_DIGEST:-}" require_ecr() { @@ -58,8 +77,8 @@ resolve_image_digest() { local cached_repository cached_tag cached_repository="$(sed -n '1p' "$OUT_DIR/image-repository" 2>/dev/null || true)" cached_tag="$(sed -n '1p' "$OUT_DIR/image-tag" 2>/dev/null || true)" - if [ "$cached_repository" != "$ECR_URI" ] || [ "$cached_tag" != "$IMAGE_TAG" ]; then - echo "Cached image digest does not belong to ECR_URI=$ECR_URI IMAGE_TAG=$IMAGE_TAG; run push first or set IMAGE_DIGEST explicitly." >&2 + if [ "$cached_repository" != "$ECR_URI" ] || [ "$cached_tag" != "$PUBLISHED_TAG" ]; then + echo "Cached image digest does not belong to ECR_URI=$ECR_URI PUBLISHED_TAG=$PUBLISHED_TAG; run push first or set IMAGE_DIGEST explicitly." >&2 exit 1 fi IMAGE_DIGEST="$(sed -n '1p' "$OUT_DIR/image-digest")" @@ -76,12 +95,12 @@ resolve_image_digest() { do_build() { local tags=(-t "$LOCAL_TAG") if [ -n "$ECR_URI" ]; then - tags+=(-t "$ECR_URI:$IMAGE_TAG") + tags+=(-t "$ECR_URI:$PUBLISHED_TAG") fi - echo ">> buildx arm64 lambda-microvm-runner (${LOCAL_TAG})" + echo ">> buildx arm64 ${MICROVM_IMAGE_TARGET} (${LOCAL_TAG})" docker buildx build \ --platform linux/arm64 \ - --target lambda-microvm-runner \ + --target "$MICROVM_IMAGE_TARGET" \ -f api/Dockerfile \ "${tags[@]}" \ --load \ @@ -91,13 +110,13 @@ do_build() { do_push() { require_ecr mkdir -p "$OUT_DIR" - echo ">> pushing $ECR_URI:$IMAGE_TAG" + echo ">> pushing $ECR_URI:$PUBLISHED_TAG" aws ecr get-login-password --region "${AWS_REGION:?AWS_REGION required}" \ | docker login --username AWS --password-stdin "${ECR_URI%%/*}" # `build` is intentionally usable without AWS/ECR configuration. Tag here as # well so a later, separately invoked `push` stage still has the remote tag. - docker image tag "$LOCAL_TAG" "$ECR_URI:$IMAGE_TAG" - docker push "$ECR_URI:$IMAGE_TAG" | tee "$OUT_DIR/push.log" + docker image tag "$LOCAL_TAG" "$ECR_URI:$PUBLISHED_TAG" + docker push "$ECR_URI:$PUBLISHED_TAG" | tee "$OUT_DIR/push.log" IMAGE_DIGEST="$(sed -n 's/^.*digest: \(sha256:[0-9a-f]\{64\}\).*$/\1/p' "$OUT_DIR/push.log" | tail -n 1)" [ -n "$IMAGE_DIGEST" ] || { echo "Could not determine the pushed ECR digest; refusing to render a mutable artifact." >&2 @@ -105,8 +124,8 @@ do_push() { } printf '%s\n' "$IMAGE_DIGEST" > "$OUT_DIR/image-digest" printf '%s\n' "$ECR_URI" > "$OUT_DIR/image-repository" - printf '%s\n' "$IMAGE_TAG" > "$OUT_DIR/image-tag" - echo ">> immutable runner ref: $ECR_URI@$IMAGE_DIGEST" + printf '%s\n' "$PUBLISHED_TAG" > "$OUT_DIR/image-tag" + echo ">> immutable ${ARTIFACT_KIND} ref: $ECR_URI@$IMAGE_DIGEST" } do_zip() { @@ -118,7 +137,7 @@ FROM ${ECR_URI}@${IMAGE_DIGEST} EOF (cd "$OUT_DIR" && rm -f artifact.zip && zip -q artifact.zip Dockerfile) printf '%s\n' "$ECR_URI" > "$OUT_DIR/artifact-image-repository" - printf '%s\n' "$IMAGE_TAG" > "$OUT_DIR/artifact-image-tag" + printf '%s\n' "$PUBLISHED_TAG" > "$OUT_DIR/artifact-image-tag" printf '%s\n' "$IMAGE_DIGEST" > "$OUT_DIR/artifact-image-digest" file_sha256 "$OUT_DIR/artifact.zip" > "$OUT_DIR/artifact-sha256" echo ">> wrote $OUT_DIR/artifact.zip (FROM ${ECR_URI}@${IMAGE_DIGEST})" @@ -145,13 +164,13 @@ do_upload() { artifact_hash="$(sed -n '1p' "$OUT_DIR/artifact-sha256")" actual_hash="$(file_sha256 "$OUT_DIR/artifact.zip")" if [ "$artifact_repository" != "$ECR_URI" ] \ - || [ "$artifact_tag" != "$IMAGE_TAG" ] \ + || [ "$artifact_tag" != "$PUBLISHED_TAG" ] \ || [ "$artifact_digest" != "$IMAGE_DIGEST" ] \ || [ "$artifact_hash" != "$actual_hash" ]; then echo "artifact.zip provenance does not match the current repository, tag, digest, or bytes; run zip again before upload." >&2 exit 1 fi - local key="$S3_URI/runner-${IMAGE_TAG}.zip" + local key="$S3_URI/${ARTIFACT_KIND}-${IMAGE_TAG}.zip" aws s3 cp "$OUT_DIR/artifact.zip" "$key" --region "${AWS_REGION:?AWS_REGION required}" echo ">> uploaded $key" cat < \\ --region \${AWS_REGION} \\