From 4d94b87881d96a385681ce7e8adbf8c1ac8f5f49 Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Thu, 3 Sep 2026 09:40:50 +0200 Subject: [PATCH 01/13] Integrate @diffusionstudio/dapi into CLI and desktop applications - Added @diffusionstudio/dapi as a dependency in package-lock.json, apps/cli/package.json, apps/desktop/package.json, and apps/web/package.json. - Refactored imports in CLI and desktop source files to utilize types and constants from @diffusionstudio/dapi, enhancing code modularity and maintainability. - Removed the cli-channels.ts file as its functionality is now encapsulated within the new dapi package. - Introduced a new MCP server design proposal in docs/mcp-server.md, outlining the transition to a unified API for tool interactions. --- apps/cli/package.json | 6 +- apps/cli/src/cli-channels.ts | 194 --------- apps/cli/src/cli-client.ts | 2 +- apps/cli/src/index.ts | 6 +- apps/cli/src/protocol.ts | 38 +- apps/desktop/package.json | 3 +- apps/desktop/src/cli-server.ts | 3 +- apps/desktop/src/main-channels.ts | 2 +- apps/desktop/src/main.ts | 2 +- apps/desktop/src/preload.ts | 2 +- apps/web/package.json | 3 +- apps/web/src/context/dapi/capture.ts | 2 +- apps/web/src/context/dapi/check.ts | 2 +- apps/web/src/context/dapi/export.ts | 2 +- apps/web/src/context/dapi/logs.ts | 2 +- apps/web/src/context/dapi/media.ts | 2 +- apps/web/src/context/dapi/models.ts | 2 +- apps/web/src/lib/ipc.ts | 4 +- docs/mcp-server.md | 216 ++++++++++ package-lock.json | 392 ++++++++++++++++++ packages/dapi/README.md | 15 + packages/dapi/package.json | 30 ++ packages/dapi/src/catalog.test.ts | 42 ++ packages/dapi/src/catalog.ts | 76 ++++ packages/dapi/src/errors.ts | 42 ++ packages/dapi/src/index.ts | 79 ++++ packages/dapi/src/schemas.ts | 97 +++++ .../dapi/src/socket.ts | 4 +- packages/dapi/src/time.test.ts | 52 +++ packages/dapi/src/time.ts | 30 ++ packages/dapi/src/tool.ts | 36 ++ packages/dapi/src/tools/capture.ts | 26 ++ packages/dapi/src/tools/check.ts | 49 +++ packages/dapi/src/tools/context.ts | 54 +++ packages/dapi/src/tools/export.ts | 61 +++ packages/dapi/src/tools/fetch.ts | 22 + packages/dapi/src/tools/fonts.ts | 34 ++ packages/dapi/src/tools/logs.ts | 20 + packages/dapi/src/tools/media-filmstrip.ts | 27 ++ packages/dapi/src/tools/media-grab.ts | 74 ++++ packages/dapi/src/tools/media-listen.ts | 36 ++ packages/dapi/src/tools/media-probe.ts | 39 ++ packages/dapi/src/tools/media-transcribe.ts | 28 ++ packages/dapi/src/tools/media-waveform.ts | 21 + packages/dapi/src/tools/models.ts | 29 ++ packages/dapi/src/tools/open.ts | 22 + packages/dapi/src/tools/report.ts | 24 ++ packages/dapi/src/tools/screenshot.ts | 20 + packages/dapi/src/tools/tools.test.ts | 110 +++++ packages/dapi/src/tools/voices.ts | 21 + packages/dapi/src/tools/whoami.ts | 15 + packages/dapi/tsconfig.json | 9 + 52 files changed, 1905 insertions(+), 224 deletions(-) delete mode 100644 apps/cli/src/cli-channels.ts create mode 100644 docs/mcp-server.md create mode 100644 packages/dapi/README.md create mode 100644 packages/dapi/package.json create mode 100644 packages/dapi/src/catalog.test.ts create mode 100644 packages/dapi/src/catalog.ts create mode 100644 packages/dapi/src/errors.ts create mode 100644 packages/dapi/src/index.ts create mode 100644 packages/dapi/src/schemas.ts rename apps/cli/src/cli-socket-path.ts => packages/dapi/src/socket.ts (80%) create mode 100644 packages/dapi/src/time.test.ts create mode 100644 packages/dapi/src/time.ts create mode 100644 packages/dapi/src/tool.ts create mode 100644 packages/dapi/src/tools/capture.ts create mode 100644 packages/dapi/src/tools/check.ts create mode 100644 packages/dapi/src/tools/context.ts create mode 100644 packages/dapi/src/tools/export.ts create mode 100644 packages/dapi/src/tools/fetch.ts create mode 100644 packages/dapi/src/tools/fonts.ts create mode 100644 packages/dapi/src/tools/logs.ts create mode 100644 packages/dapi/src/tools/media-filmstrip.ts create mode 100644 packages/dapi/src/tools/media-grab.ts create mode 100644 packages/dapi/src/tools/media-listen.ts create mode 100644 packages/dapi/src/tools/media-probe.ts create mode 100644 packages/dapi/src/tools/media-transcribe.ts create mode 100644 packages/dapi/src/tools/media-waveform.ts create mode 100644 packages/dapi/src/tools/models.ts create mode 100644 packages/dapi/src/tools/open.ts create mode 100644 packages/dapi/src/tools/report.ts create mode 100644 packages/dapi/src/tools/screenshot.ts create mode 100644 packages/dapi/src/tools/tools.test.ts create mode 100644 packages/dapi/src/tools/voices.ts create mode 100644 packages/dapi/src/tools/whoami.ts create mode 100644 packages/dapi/tsconfig.json diff --git a/apps/cli/package.json b/apps/cli/package.json index 994710b0..76346cd6 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -7,8 +7,7 @@ "dapi": "./dist/index.js" }, "exports": { - "./protocol": "./src/protocol.ts", - "./channels": "./src/cli-channels.ts" + "./protocol": "./src/protocol.ts" }, "scripts": { "build": "esbuild src/index.ts --bundle --platform=node --format=cjs --external:esbuild --external:@babel/core --external:@babel/preset-typescript --external:babel-preset-solid --external:bufferutil --external:utf-8-validate --outfile=dist/index.js && chmod +x dist/index.js", @@ -24,7 +23,8 @@ "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", "esbuild": "^0.28.1", - "ws": "^8.18.3" + "ws": "^8.18.3", + "@diffusionstudio/dapi": "*" }, "devDependencies": { "@types/babel__core": "^7.20.5", diff --git a/apps/cli/src/cli-channels.ts b/apps/cli/src/cli-channels.ts deleted file mode 100644 index adecc9ca..00000000 --- a/apps/cli/src/cli-channels.ts +++ /dev/null @@ -1,194 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -// Wire-level channel for the CLI handshake. Each CLI command hosts a -// short-lived WebSocket server; main's only job is to relay the connect -// info to the renderer, which then dials the CLI directly. Main never sees -// request payloads. -export const CLI_WIRE = { - CONNECT: "cli:connect", -} as const; - -// Sent by the CLI to main over the unix socket, relayed verbatim to the -// renderer. The token guards the loopback WebSocket server against other -// local processes racing to connect first. -export type CliHandshake = { port: number; token: string }; - -export type CliHandshakeReply = { ok: true } | { ok: false; error: string }; - -// One tRPC request/reply pair per WebSocket connection. `path` is the -// dot-joined procedure path in the renderer's router (e.g. "media.frame"); -// procedure inputs and outputs are typed end-to-end via the AppRouter type, -// so the wire envelope stays untyped. -export type CliRequest = { - path: string; - input: unknown; -}; - -export type CliReply = - | { ok: true; data: unknown } - | { ok: false; error: string }; - -export type AssetRef = { path: string }; - -export type MediaProbeRequest = AssetRef; - -export type FrameQuality = "small" | "medium" | "large" | "fullres"; -export type MediaFrameRequest = AssetRef & { - times?: number[]; - count?: number; - start?: number; - end?: number; - quality?: FrameQuality; - auto?: boolean; - combine?: boolean; - perSheet?: number; -}; - -/** Beyond this the cells get too small to be worth the tokens; use `filmstrip`. */ -export const MAX_FRAMES_PER_SHEET = 12; - -/** - * One written image: a single frame stamped with its timecode, or a contact - * sheet stamped with the span it covers (`0f-08s10f`). - */ -export type TimecodedImage = { timecode: string; base64: string }; - -export type MediaFrameResult = TimecodedImage[]; - -export type CaptureRequest = { - id: string; - frames?: number[]; - combine?: boolean; - perSheet?: number; -}; - -export type CaptureResult = TimecodedImage[]; - -export type MediaTranscribeRequest = AssetRef; -export type TranscriptWord = { text: string; start: number; end: number }; -export type TranscriptSegment = { text: string; words: TranscriptWord[] }; -export type MediaTranscribeResult = { segments: TranscriptSegment[] }; - -export type MediaFilmstripRequest = AssetRef & { start?: number; end?: number; scale?: number }; -export type MediaFilmstripResult = { base64: string }; - -export type MediaWaveformRequest = AssetRef & { start?: number; end?: number; scale?: number }; -export type MediaWaveformResult = { - base64: string; - silences: Array<{ start: number; end: number }>; -}; - -export type MediaListenRequest = AssetRef & { prompt?: string; start?: number; end?: number; stripVideo?: boolean }; -export type MediaListenResult = { result?: string; start?: number; end?: number }; - -export type CheckRequest = { id: string }; - -export type CheckIssueCode = - | "black-frames" - | "no-visuals" - | "never-visible" - | "zero-duration" - | "transparent" - | "source-error"; - -/** - * One structural finding. `ranges` (where present) are seconds relative to - * the checked node's start — the same clock `capture --time` uses. - */ -export type CheckIssue = { - code: CheckIssueCode; - severity: "error" | "warning"; - message: string; - /** Source stamp of the offending node; absent when the issue is about the subtree as a whole. */ - node?: string; - ranges?: Array<{ start: number; end: number }>; -}; - -export type CheckResult = { - stats: { - /** Nodes in the subtree, the checked node included. */ - nodes: number; - byKind: Record; - /** Deepest nesting level below the checked node (0 = no children). */ - depth: number; - /** Seconds the checked node plays (its workarea, when one is set). */ - duration: number; - }; - issues: CheckIssue[]; -}; - -export type ExportFormat = "mp4" | "webm" | "ogg" | "mov"; - -// The settings shape mirrors the scene's `diffusion.export.` entry in the -// project's package.json (see the web app's engine/project-config), spelled -// out here so the wire seam stays dependency-free. Codecs are strings on the -// wire; the app validates them against what the encoder accepts. -export type ExportVideoSettings = { - enabled?: boolean; - codec?: string; - bitrate?: number; - fps?: number; - resolution?: number; -}; - -export type ExportAudioSettings = { - enabled?: boolean; - codec?: string; - sampleRate?: number; - bitrate?: number; -}; - -export type ExportSettings = { - format?: ExportFormat; - video?: ExportVideoSettings; - audio?: ExportAudioSettings; -}; - -/** - * `path` is the absolute output file, whose extension picks the container; - * omitted, the app writes `exports/.` in the project folder. - * Everything else — codecs, bitrates, resolution — is read from the scene's - * export entry in the project's package.json. - */ -export type ExportRequest = { id: string; path?: string }; - -/** - * `config` echoes the settings the export was made with — the package.json - * entry (or the defaults), with the container the extension resolved to — so - * a caller sees what its config edit actually did. `width`/`height` are the - * encoded pixel size (0×0 for an audio-only export); `duration` is seconds. - */ -export type ExportResult = { - path: string; - width: number; - height: number; - duration: number; - size: number; - config: ExportSettings; -}; - -export type GeneratedAsset = { id: string; name: string; type: string }; - -export type ModelsRequest = { type?: "image" | "video" | "audio" }; - -export type ModelInfo = { - type: "image" | "video" | "audio"; - id: string; - name: string; - durations?: string[]; - aspectRatios?: string[]; - features?: Array<"start-frame" | "end-frame" | "audio">; -}; - -export type VoiceInfo = { id: string; label: string; description: string }; - -export type ScreenshotResult = { base64: string; width: number; height: number }; - -export type LogLevel = "debug" | "info" | "warning" | "error"; - -export type LogEntry = { ts: number; level: LogLevel; message: string; source: string }; - -export type LogsRequest = { tail?: number; level?: LogLevel }; - diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts index fc125a02..6554862f 100644 --- a/apps/cli/src/cli-client.ts +++ b/apps/cli/src/cli-client.ts @@ -9,7 +9,7 @@ import { WebSocketServer } from "ws"; import { createTRPCClient, TRPCClientError } from "@trpc/client"; import type { TRPCLink } from "@trpc/client"; import { observable } from "@trpc/server/observable"; -import { SOCKET_PATH } from "./protocol"; +import { SOCKET_PATH } from "@diffusionstudio/dapi/socket"; import type { CliHandshake, CliHandshakeReply, CliReply, CliRequest } from "./protocol"; import type { AppRouter } from "../../web/src/context/dapi"; diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 184dac9b..e8e74f82 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -15,8 +15,8 @@ import { editor, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, waitForCliSo import { listLocalFonts } from "./fonts"; import { buildIssueBody, createIssue } from "./report"; import { fetchVideo } from "./ytdlp"; -import { MAX_FRAMES_PER_SHEET } from "./protocol"; -import type { AssetRef, FrameQuality, LogEntry, LogLevel, TimecodedImage } from "./protocol"; +import { MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi"; +import type { FrameQuality, LogEntry, LogLevel, TimecodedImage } from "@diffusionstudio/dapi"; // Long-running commands (renders, AI generation) override the default 60s. const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } }; @@ -129,7 +129,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { * anything else — a URL, or a library path (`b-roll/clip.mp4`) — is passed * through for the app to resolve. Library paths need an open project. */ -function resolveAssetRef(ref: string): AssetRef { +function resolveAssetRef(ref: string): { path: string } { const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref); if (existsSync(absPath)) return { path: absPath }; if (isAbsolute(ref)) { diff --git a/apps/cli/src/protocol.ts b/apps/cli/src/protocol.ts index d4d60007..9119d703 100644 --- a/apps/cli/src/protocol.ts +++ b/apps/cli/src/protocol.ts @@ -2,10 +2,34 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// Public wire protocol consumed by both the CLI binary and any host that -// embeds the CLI socket server (the Diffusion Studio desktop app). Importing -// from `@diffusionstudio/cli/protocol` is the OSS-friendly seam: this module has no -// dependency on proprietary code, so the package stays extractable into its -// own repo. -export * from "./cli-channels"; -export * from "./cli-socket-path"; +// The transport between the CLI and the app, and nothing else: the tool +// catalog and every request and result type live in @diffusionstudio/dapi. +// Free of Node built-ins so the renderer can import it; the socket path is +// in @diffusionstudio/dapi/socket. +// +// Each CLI command hosts a short-lived WebSocket server; main's only job is +// to relay the connect info to the renderer, which then dials the CLI +// directly. Main never sees request payloads. +export const CLI_WIRE = { + CONNECT: "cli:connect", +} as const; + +// Sent by the CLI to main over the unix socket, relayed verbatim to the +// renderer. The token guards the loopback WebSocket server against other +// local processes racing to connect first. +export type CliHandshake = { port: number; token: string }; + +export type CliHandshakeReply = { ok: true } | { ok: false; error: string }; + +// One tRPC request/reply pair per WebSocket connection. `path` is the +// dot-joined procedure path in the renderer's router (e.g. "media.frame"); +// procedure inputs and outputs are typed end-to-end via the AppRouter type, +// so the wire envelope stays untyped. +export type CliRequest = { + path: string; + input: unknown; +}; + +export type CliReply = + | { ok: true; data: unknown } + | { ok: false; error: string }; diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 20c0d2ba..56a49222 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -44,6 +44,7 @@ "nanoid": "^6.0.1", "ts-morph": "^28.0.0", "update-electron-app": "^3.0.0", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "@diffusionstudio/dapi": "*" } } diff --git a/apps/desktop/src/cli-server.ts b/apps/desktop/src/cli-server.ts index c78cf301..457cb750 100644 --- a/apps/desktop/src/cli-server.ts +++ b/apps/desktop/src/cli-server.ts @@ -6,7 +6,8 @@ import { existsSync, unlinkSync } from "node:fs"; import { createServer } from "node:net"; import type { Server, Socket } from "node:net"; import { app, BrowserWindow } from "electron"; -import { CLI_WIRE, SOCKET_PATH } from "@diffusionstudio/cli/protocol"; +import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; +import { SOCKET_PATH } from "@diffusionstudio/dapi/socket"; import type { CliHandshake, CliHandshakeReply } from "@diffusionstudio/cli/protocol"; import { mainBridge } from "./main-manager"; import { MAIN_CHANNELS } from "./main-channels"; diff --git a/apps/desktop/src/main-channels.ts b/apps/desktop/src/main-channels.ts index 5df8bb83..9fd04403 100644 --- a/apps/desktop/src/main-channels.ts +++ b/apps/desktop/src/main-channels.ts @@ -9,7 +9,7 @@ // // CLI traffic uses a separate wire pair (CLI_WIRE in @diffusionstudio/cli/protocol); // main forwards it opaquely without inspecting channel names. -import type { LogEntry, ScreenshotResult } from "@diffusionstudio/cli/protocol"; +import type { LogEntry, ScreenshotResult } from "@diffusionstudio/dapi"; import type { SourceEdit, WriteResult } from "./edit"; export const MAIN_WIRE = { diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index b965b054..ad16a026 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -42,7 +42,7 @@ import { writeProject, } from "./projects"; import type { DeepLinkChannel } from "./main-channels"; -import type { LogEntry } from "@diffusionstudio/cli/protocol"; +import type { LogEntry } from "@diffusionstudio/dapi"; const DEV_URL = "http://localhost:5173"; const AUTH_PROTOCOL = "diffusion"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 688361ec..60e10afe 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -4,7 +4,7 @@ import { contextBridge, ipcRenderer, webUtils } from "electron"; import { MAIN_WIRE } from "./main-channels"; -import { CLI_WIRE } from "@diffusionstudio/cli/channels"; +import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; import type { IpcRendererEvent } from "electron"; diff --git a/apps/web/package.json b/apps/web/package.json index 619eba2a..a60ab03c 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -42,7 +42,8 @@ "stats.js": "^0.17.0", "tailwind-merge": "^3.4.0", "typegpu": "^0.11.9", - "zod": "^4.2.1" + "zod": "^4.2.1", + "@diffusionstudio/dapi": "*" }, "devDependencies": { "@eslint/js": "^9.39.2", diff --git a/apps/web/src/context/dapi/capture.ts b/apps/web/src/context/dapi/capture.ts index 28034ae0..375e7bae 100644 --- a/apps/web/src/context/dapi/capture.ts +++ b/apps/web/src/context/dapi/capture.ts @@ -16,7 +16,7 @@ import { getParentNode, isScene, Source } from "@diffusionstudio/runtime"; import { createCapture } from "@/engine/capture"; import { resolveNode } from "./nodes"; -import type { CaptureRequest, CaptureResult, TimecodedImage } from "@diffusionstudio/cli/channels"; +import type { CaptureRequest, CaptureResult, TimecodedImage } from "@diffusionstudio/dapi"; import type { EditorSession } from "./session"; // Ceiling on the height a sheet cell renders a node at. diff --git a/apps/web/src/context/dapi/check.ts b/apps/web/src/context/dapi/check.ts index 15a59505..0aabb6df 100644 --- a/apps/web/src/context/dapi/check.ts +++ b/apps/web/src/context/dapi/check.ts @@ -10,7 +10,7 @@ import { import { resolveNode } from "./nodes"; -import type { CheckIssue, CheckRequest, CheckResult } from "@diffusionstudio/cli/channels"; +import type { CheckIssue, CheckRequest, CheckResult } from "@diffusionstudio/dapi"; import type { Entity } from "koota"; import type { EditorSession } from "./session"; diff --git a/apps/web/src/context/dapi/export.ts b/apps/web/src/context/dapi/export.ts index a2a6536e..c4e96e85 100644 --- a/apps/web/src/context/dapi/export.ts +++ b/apps/web/src/context/dapi/export.ts @@ -18,7 +18,7 @@ import { mainBridge } from "@/lib/ipc"; import { MAIN_CHANNELS } from "@desktop/main-channels"; import { resolveNode } from "./nodes"; -import type { ExportRequest, ExportResult, ExportSettings } from "@diffusionstudio/cli/channels"; +import type { ExportRequest, ExportResult, ExportSettings } from "@diffusionstudio/dapi"; import type { ContainerFormat, ExportConfig } from "@/engine/project-config"; import type { EditorSession } from "./session"; diff --git a/apps/web/src/context/dapi/logs.ts b/apps/web/src/context/dapi/logs.ts index e70d29e5..e45d2389 100644 --- a/apps/web/src/context/dapi/logs.ts +++ b/apps/web/src/context/dapi/logs.ts @@ -4,7 +4,7 @@ import { mainBridge } from "@/lib/ipc"; import { MAIN_CHANNELS } from "@desktop/main-channels"; -import type { LogsRequest, LogLevel } from "@diffusionstudio/cli/channels"; +import type { LogsRequest, LogLevel } from "@diffusionstudio/dapi"; const LEVEL_RANK: Record = { debug: 0, info: 1, warning: 2, error: 3 }; diff --git a/apps/web/src/context/dapi/media.ts b/apps/web/src/context/dapi/media.ts index 74cab872..09c643f9 100644 --- a/apps/web/src/context/dapi/media.ts +++ b/apps/web/src/context/dapi/media.ts @@ -16,7 +16,7 @@ import { createProjectFS } from '@/projects/fs'; import type { Accessor } from 'solid-js'; import type { Asset } from '@diffusionstudio/assets'; import type { EditorSession } from './session'; -import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/cli/channels"; +import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/dapi"; type ResolveAsset = (path: string) => Promise; diff --git a/apps/web/src/context/dapi/models.ts b/apps/web/src/context/dapi/models.ts index 9a63257e..2b5b07f5 100644 --- a/apps/web/src/context/dapi/models.ts +++ b/apps/web/src/context/dapi/models.ts @@ -7,7 +7,7 @@ import { PROMPT_INPUT_VIDEO_MODEL_OPTIONS, PROMPT_INPUT_AUDIO_MODEL_OPTIONS, } from "@/components/genai/config"; -import type { ModelsRequest, ModelInfo } from "@diffusionstudio/cli/channels"; +import type { ModelsRequest, ModelInfo } from "@diffusionstudio/dapi"; export function handleModels() { return async (req: ModelsRequest) => { diff --git a/apps/web/src/lib/ipc.ts b/apps/web/src/lib/ipc.ts index 98b61761..54447eaf 100644 --- a/apps/web/src/lib/ipc.ts +++ b/apps/web/src/lib/ipc.ts @@ -12,8 +12,8 @@ import type { MainRequestChannel, MainRequestMap, } from "@desktop/main-channels"; -import { CLI_WIRE } from "@diffusionstudio/cli/channels"; -import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/channels"; +import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; +import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol"; type EventHandler = (data: MainEventMap[C]) => void; diff --git a/docs/mcp-server.md b/docs/mcp-server.md new file mode 100644 index 00000000..5b93ba17 --- /dev/null +++ b/docs/mcp-server.md @@ -0,0 +1,216 @@ +# Design: an MCP server in the app, with the CLI as its client + +| | | +| --- | --- | +| Status | Proposal | +| Branch | `mcp-support` | +| Date | 2026-09-02 | +| Scope | `apps/desktop`, `apps/web`, `apps/cli`, a new `packages/dapi`, the editor skill and `reference/` | + +## Summary + +Today every `dapi` command is a client of nothing and a server of something: the CLI process opens its own WebSocket server, asks the Electron main process over a unix socket to tell the renderer about it, and the renderer dials back and answers one tRPC call. This document proposes the ordinary direction instead. The app hosts one persistent [MCP](https://modelcontextprotocol.io) server in the Electron main process. Agents connect to it directly. The `dapi` binary shrinks to a launcher, a stdio-to-socket pipe for agents, and a thin human-facing client. The tool handlers stay exactly where they are, in the renderer, because that is where the project world, the engine, WebCodecs and WebGPU live. + +The point is not MCP for its own sake. The point is that the API gets one definition instead of three, the app becomes reachable without an admin-prompt install step, long operations get progress and cancellation, and the CLI stops needing to match the app's version. + +## Goals + +- One definition of the API: tool names, input schemas, descriptions and result shapes, owned by the app and served to clients at runtime. +- Agents reach the app with a one-line registration and no PATH install. +- Long operations (export, generation, `listen`) report progress and can be cancelled, without client-side timeouts. +- The CLI binary stays useful for humans, scripts and CI, and no longer has to be built against a specific router type. +- Keep the renderer handlers and their behaviour. This is a transport and surface change, not a rewrite of `capture`, `check`, `export` or the media handlers. + +## Non-goals + +- A network-reachable server. Everything stays on the local machine, for the local user. +- Changing the web (browser-only) app. It has no `window.desktop` and no CLI today; that stays true. +- Moving media decoding or rendering into the main process. +- Multi-window routing. One window answers, as today. + +## The system today + +```mermaid +sequenceDiagram + participant CLI as dapi (per command) + participant Main as Electron main + participant R as Renderer + CLI->>CLI: start WebSocket server on 127.0.0.1:random + CLI->>Main: unix socket: {port, token} + Main->>R: IPC cli:connect {port, token} + Main-->>CLI: {ok: true} + R->>CLI: ws://127.0.0.1:port?token=… + CLI->>R: {path: "media.frame", input} + R->>R: tRPC caller → handler + R-->>CLI: {ok, data} + CLI->>CLI: write PNGs, print JSON, exit +``` + +The pieces, for reference: + +- `apps/cli/src/cli-client.ts` hosts the per-command WebSocket server and wraps it in a tRPC link typed against `AppRouter` from `apps/web`. +- `apps/cli/src/index.ts` is a commander program: every command re-describes its arguments and options in help text, validates them, calls the typed client, and formats the result. +- `apps/cli/src/cli-channels.ts` hand-writes the wire types for every request and result. +- `apps/desktop/src/cli-server.ts` listens on `SOCKET_PATH`, waits for the renderer to finish loading, relays the handshake, and switches the app into headless mode on the first connection. +- `apps/web/src/lib/ipc.ts` (`CliBridge`) receives the handshake, dials the CLI, dispatches one request to whichever registered router owns the path, and holds requests that arrive while no router is mounted. +- `apps/web/src/context/dapi/api.tsx` builds the tRPC router over the handlers in the same folder. Inputs are cast, not validated (`apps/web/src/lib/cli-rpc.ts`). + +### What is wrong with it + +1. **The direction is inverted.** The client hosts the server. That forces the three-hop handshake, a random port, a token to guard it, and a fresh server per call. Main exists in the path only to pass along a port number. +2. **Three descriptions of one API.** Commander help text, the tRPC router's inferred types, and the hand-written wire types all describe the same procedures and can drift. The editor skill tells agents to treat `dapi --help` as authoritative, so the help text is effectively the public schema, maintained by hand. +3. **Version coupling.** The CLI is compiled against one `AppRouter`. A CLI symlinked from a different app build than the one running is silently wrong at the type level and possibly at runtime. The wire has no runtime validation. +4. **No progress, no cancellation.** The client picks a timeout per call (60 s, 10 min for generation, 1 h for export) and there is no way to stop an export once started. +5. **Install friction.** Nothing works until `/usr/local/bin/dapi` exists, which needs the macOS admin prompt in `apps/desktop/src/cli-install.ts`. Every agent session begins by checking that. +6. **Local commands live in the wrong process.** `fonts`, `fetch` (yt-dlp) and `report` run in the CLI process and are unavailable to anything that is not the CLI. + +## Proposed architecture + +```mermaid +sequenceDiagram + participant Agent as Agent (Claude Code, …) + participant Proxy as dapi mcp (stdio pipe) + participant Main as Electron main: MCP server + participant R as Renderer: tool handlers + Agent->>Proxy: spawn; JSON-RPC over stdio + Proxy->>Main: connect unix socket (launch app if absent) + Proxy->>Main: bytes, unchanged + Main->>Main: initialize, tools/list from catalog + Agent->>Main: tools/call capture {id, times} + Main->>R: IPC dapi:call {callId, tool, input} + R-->>Main: IPC dapi:progress {callId, …} (optional) + R-->>Main: IPC dapi:reply {callId, ok, data} + Main->>Main: write PNGs, build content blocks + Main-->>Agent: result {content, structuredContent} +``` + +Four parts. + +### 1. `packages/dapi`: the tool catalog + +A new workspace package, `@diffusionstudio/dapi`, that main, the renderer and the CLI all import. It replaces `apps/cli/src/cli-channels.ts` and the `./protocol` and `./channels` exports of the CLI package. + +It exports one catalog: for every tool, its name, description, a zod input schema, a zod output schema for the structured result, and where it runs (`renderer` or `main`). The descriptions are the text agents read in `tools/list`, so the long, careful command descriptions currently in `apps/cli/src/index.ts` move here verbatim. The package's main export is free of Node built-ins so the renderer can import it; the `SOCKET_PATH` constant sits behind `@diffusionstudio/dapi/socket`. The IPC envelope types for the main-to-renderer call channel join the package in step 3, when something uses them. + +zod is already a dependency of `apps/web`, and the MCP TypeScript SDK takes zod schemas directly, so there is no separate JSON Schema to maintain. + +Tools, mapped from today's commands: + +| Tool | Today | Runs in | Notes | +| --- | --- | --- | --- | +| `open` | `dapi open ` | renderer | Opens or creates a project and navigates to it. Launching the app is the proxy's job, see part 3. | +| `context` | `dapi context` | renderer | | +| `capture` | `dapi capture` | renderer | Result handling in part 2. | +| `check` | `dapi check` | renderer | | +| `export` | `dapi export` | renderer | Progress and cancellation. | +| `models`, `voices`, `whoami` | same | renderer | | +| `logs` | `dapi logs` | main | Main owns the log buffer already; the renderer handler only forwards to it. | +| `screenshot` | `dapi screenshot` | renderer | | +| `media_probe`, `media_grab`, `media_transcribe`, `media_filmstrip`, `media_waveform`, `media_listen` | `dapi media …` | renderer | `grab` keeps its frame cap and `uncapped` flag as inputs. | +| `fonts` | `dapi fonts` | main | Moves from the CLI process to main. Same code. | +| `fetch` | `dapi fetch` | main | yt-dlp passthrough. Progress goes to MCP progress notifications instead of inherited stderr. | +| `report` | `dapi report` | main | Files the GitHub issue with diagnostics that main already holds. | + +Argument parsing that exists today only to turn strings into numbers (`--time "45f"`, `--count`, `--per-sheet`) becomes schema: times are accepted as the same strings and parsed by `parseTime` from `@diffusionstudio/jsx` inside the schema's transform, so the CLI and the agent get identical validation and identical error messages. + +### 2. Main process: the server + +`apps/desktop/src/cli-server.ts` is replaced by `mcp-server.ts`. It still listens on `SOCKET_PATH` (`/tmp/diffusion-studio.sock` on macOS and Linux, `\\.\pipe\diffusion-studio` on Windows), still cleans up a stale socket file on start, and still enables headless mode on the first connection. What changes is what it speaks. + +**Framing.** Each accepted socket carries newline-delimited JSON-RPC, exactly the MCP stdio framing. The SDK exports the `ReadBuffer` and `serializeMessage` helpers used by its stdio transport, so the socket transport is a short class around them. One `McpServer` instance per connection, because the SDK binds one server to one transport. Several agents can be connected at once; each gets its own session. + +**Authentication.** None beyond the socket. On macOS the socket lives in the per-user temp directory; on Linux `/tmp` is shared, so the file is created with mode `0600`. The token that guards today's WebSocket server exists only because that server listens on TCP; it is not needed. + +**Registration.** On startup main iterates the catalog and calls `registerTool` for each entry. A `main` tool calls its handler directly. A `renderer` tool forwards: + +```ts +// apps/desktop/src/mcp-server.ts, sketch +for (const tool of catalog) { + server.registerTool(tool.name, { + description: tool.description, + inputSchema: tool.input, + outputSchema: tool.output, + }, async (input, extra) => { + const data = tool.runsIn === "main" + ? await mainHandlers[tool.name](input, extra) + : await callRenderer(tool.name, input, extra); // IPC, below + return present(tool, data); // content + structuredContent + }); +} +``` + +**Main-to-renderer calls.** The current bridge (`apps/desktop/src/main-manager.ts` and `MainBridge` in `apps/web/src/lib/ipc.ts`) carries requests from the renderer to main and events from main to the renderer. It has no request in the other direction, so this adds one, with three IPC messages: + +- `dapi:call` from main: `{ callId, tool, input }` +- `dapi:progress` from renderer: `{ callId, progress, total?, message? }` +- `dapi:reply` from renderer: `{ callId, ok, data | error }` +- `dapi:cancel` from main: `{ callId }` + +Main keeps a map of in-flight calls. Before sending it waits for the renderer to finish loading, which `waitForRendererReady` already does. In the renderer, `CliBridge` loses its WebSocket code and keeps its dispatch and hold-until-a-handler-registers logic, now fed by `dapi:call`. The renderer registers a plain `Record` instead of a tRPC router; each handler receives the already-validated input, an `AbortSignal`, and a `report(progress)` callback. tRPC and its `cli-rpc.ts` helpers go away. + +**Progress and cancellation.** When the client passes a `progressToken`, main turns `dapi:progress` into `notifications/progress`. When the client sends `notifications/cancelled`, main sends `dapi:cancel` and the handler's `AbortSignal` fires. Client-side timeouts disappear; the client decides how long it is willing to wait. `export` reports encoded frames; `media_listen` and generation-backed work report whatever stage they are in; the others report nothing and that is fine. + +**Results.** Every tool returns `structuredContent` matching its output schema, plus a text block with the same JSON so clients that ignore structured content still get it. Tools that produce images (`capture`, `media_grab`, `media_filmstrip`, `media_waveform`, `screenshot`) keep today's behaviour of writing PNGs to an output directory (an `output` input, defaulting to a fresh directory under the temp dir) and returning the paths. In addition, they inline `image` content blocks when the result is small: at most four images and none over about a megabyte. A contact sheet of a few positions therefore arrives in the agent's context immediately, while an uncapped hundred-frame grab arrives as a directory the agent reads selectively, which is the behaviour the current CLI was designed for. + +This does mean main sees payloads, which the current design deliberately avoids. The cost is one structured-clone copy over Electron IPC per call. Frame data crosses as `Uint8Array` rather than base64 strings to keep that copy cheap; base64 encoding happens once, in main, only for the blocks that are inlined. + +**Resources.** Two things agents poll today are better as MCP resources: `dapi://context` (what `context` returns, so a client can subscribe to changes) and `dapi://logs`. The per-project authoring reference the app writes into each project, which `AGENTS.md` points at, can be listed as a resource too, so an agent can read it without knowing the path. None of this is required for the cut-over; it is listed because it is cheap once the server exists. + +### 3. The `dapi` binary + +`apps/cli` stays, and stays bundled with the app, but its job changes. + +**`dapi mcp`** is the entry point agents register. It is a byte pipe: `process.stdin` to the socket, the socket to `process.stdout`, no JSON parsing. If the socket is absent or refused and the platform is macOS, it launches the app in the background with `open -g -a "Diffusion Studio" --args --hidden`, then retries the connection for up to 30 s, which is the existing `launchApp` and `waitForCliSocket` logic. The proxy needs no SDK dependency, and because it never interprets messages, any app version works with any proxy version. Registration is one line: + +```bash +claude mcp add dapi -- dapi mcp +``` + +or the equivalent `.mcp.json` entry. Because the app bundle ships the binary at a known path under `Contents/Resources/cli/bin/dapi`, the app can also offer that registration itself in place of today's "install CLI" button, without a PATH symlink and without the admin prompt. The symlink install stays available for people who want `dapi` in a shell. + +**`dapi open [dir]`** keeps its meaning: launch or surface the app, then call the `open` tool. + +**`dapi call [--json '{…}']`** is the generic client for scripts and debugging. It uses the SDK client over the socket, prints `structuredContent` as JSON on stdout, and writes any inlined images to disk. + +**Named wrappers** survive for the commands people type by hand in a shell or in CI: `capture`, `check`, `export`, `context`, `media probe`, `media grab`, at least. They are thin: they build the input object and call `call`. Their help text is generated from the catalog's descriptions, so `dapi --help` and `tools/list` say the same thing because they are the same string. Commands nobody types by hand (`voices`, `models`, `whoami`, `screenshot`, `logs`) are reachable through `call` and need no wrapper unless one is missed. + +Removed from the CLI package: `cli-client.ts`, `cli-channels.ts`, the tRPC and `ws` dependencies, and `fonts.ts`, `ytdlp.ts` and `report.ts`, which move to main. + +### 4. Skill and reference docs + +`apps/desktop/skills/editor/SKILL.md` currently says the CLI is self-describing via `--help`. It changes to name the MCP tools and to say that `tools/list` is authoritative, with a short fallback paragraph for environments without MCP (use `dapi call`). The per-command files in `reference/` become per-tool, and their option tables are generated from the catalog so they cannot drift either. `reference/README.md` loses the sentence about "every command talks to the running app over a local socket" and gains one about the server. + +## Behaviour that must not change + +- **Headless mode** switches on at the first connection, as now. +- **Requests during a project switch** are held and answered once the editor remounts. The queue moves from the renderer's `CliBridge` to being fed by IPC, but the semantics stay. +- **Asset resolution** for `media_*` tools keeps the rule in `createAssetResolver`: library paths need an open project, absolute paths and URLs work without one. +- **Sign-in gating** for `media_listen` and generation stays in the handler. +- **Error text.** Handlers throw with messages written for an agent to read (`No project open — run open first`). Main returns those as `isError` tool results, not JSON-RPC errors, so the agent sees the sentence. + +## Migration + +The CLI ships inside the app bundle, so there is no compatibility window to maintain between old CLIs and new apps. This can land as one feature branch in ordered steps, each of which builds and type-checks on its own. + +1. **Catalog.** Create `packages/dapi` with the tool catalog, `Time`, and `DapiError`, plus tests. Point the renderer handlers' request and result types at it. Delete `apps/cli/src/cli-channels.ts`; `apps/cli/src/protocol.ts` keeps only the handshake and envelope types of the current transport. No behaviour change. *Landed on `mcp-support`.* +2. **Renderer.** Replace the tRPC router in `api.tsx` with a handler map keyed by tool name, validated by the catalog schemas, with `AbortSignal` and progress plumbed in. Replace the WebSocket half of `CliBridge` with the `dapi:call` IPC handling. At this step the old CLI stops working; that is expected. +3. **Main.** Add `mcp-server.ts` with the socket transport, catalog registration, the in-flight map, result presentation, and the `logs`, `fonts`, `fetch` and `report` handlers. Delete `cli-server.ts`. Verify with the MCP Inspector against the socket. +4. **CLI.** Rewrite `apps/cli` as `mcp`, `open`, `call` and the wrappers. Drop tRPC and `ws`. Update `cli-install.ts` and the settings UI to offer MCP registration. +5. **Docs.** Update the skill and regenerate `reference/`. Update the installation reference the skill points to. +6. **Cleanup.** Remove `CLI_WIRE`, the handshake types, and the `./protocol` and `./channels` exports from the CLI package. + +## Open questions + +- **Inline image thresholds.** Four images and about a megabyte each is a starting point. It should be tuned against what a contact sheet at the default sizes actually weighs. +- **Windows launch.** `open -a` is macOS-only today and the proposal keeps that. Windows and Linux users get a clear "launch the app first" error, as they do now. Worth fixing, not in scope here. +- **HTTP transport later.** Streamable HTTP on loopback would let clients that cannot spawn a process connect. It needs a discovery file for the port and brings back the token. Not needed while every client we care about can run `dapi mcp`. +- **Should `open` launch the app?** The tool cannot launch its own host, so launching stays in the proxy. The alternative, a separate tiny launcher binary, adds a second thing to install for no gain. +- **Concurrency limits.** Today two `dapi` processes can run two captures at once and the renderer copes. With one server that is unchanged, but it becomes easy for an agent to fire several exports in parallel. A per-tool concurrency cap in main is cheap if it turns out to matter. + +## Alternatives considered + +**Keep tRPC, invert the transport, add `dapi mcp` as a wrapper.** Main hosts a persistent WebSocket server speaking the current envelope; the CLI adds a stdio MCP server that wraps the existing typed client. Least code changed, and it fixes the inverted direction. It keeps the three API descriptions and adds a fourth (the MCP tool schemas in the CLI), and the CLI stays version-coupled to the app. Rejected for that reason. + +**MCP over Streamable HTTP only.** Standard, works for clients that cannot spawn processes, and the SDK ships the transport. It needs a fixed or discovered port, a token, and CORS thinking. The socket transport is smaller and the proxy covers every current client; HTTP can be added beside it later. + +**Run the MCP server in the renderer.** Keeps main out of the payload path. The renderer cannot listen on a socket, so main would have to be a byte pipe into the page, and progress, cancellation and the hold-until-ready queue would all live in page code that reloads on project switch. The main-process server is simpler and more robust. diff --git a/package-lock.json b/package-lock.json index d48abb1a..f16d06a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "dependencies": { "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", + "@diffusionstudio/dapi": "*", "@trpc/client": "^11.18.0", "@trpc/server": "^11.18.0", "babel-preset-solid": "^1.9.12", @@ -49,6 +50,7 @@ "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", "@diffusionstudio/cli": "*", + "@diffusionstudio/dapi": "*", "@diffusionstudio/jsx": "*", "babel-preset-solid": "^1.9.12", "esbuild": "^0.28.1", @@ -94,6 +96,7 @@ "@diffusionstudio/api-contract": "^0.1.0", "@diffusionstudio/assets": "*", "@diffusionstudio/cli": "*", + "@diffusionstudio/dapi": "*", "@diffusionstudio/encoder": "*", "@diffusionstudio/jsx": "*", "@diffusionstudio/koota-solid": "*", @@ -623,6 +626,10 @@ "resolved": "apps/cli", "link": true }, + "node_modules/@diffusionstudio/dapi": { + "resolved": "packages/dapi", + "link": true + }, "node_modules/@diffusionstudio/desktop": { "resolved": "apps/desktop", "link": true @@ -3535,6 +3542,13 @@ "solid-js": "^1.8.6" } }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@supabase/auth-js": { "version": "2.109.0", "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.109.0.tgz", @@ -4077,6 +4091,17 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/chrome": { "version": "0.1.43", "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.1.43.tgz", @@ -4088,6 +4113,13 @@ "@types/har-format": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/dom-mediacapture-transform": { "version": "0.1.11", "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz", @@ -4502,6 +4534,119 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@vitest/expect": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.11", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.11", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.11", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@vscode/sudo-prompt": { "version": "9.3.2", "resolved": "https://registry.npmjs.org/@vscode/sudo-prompt/-/sudo-prompt-9.3.2.tgz", @@ -4951,6 +5096,16 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/ast-kit": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-2.2.0.tgz", @@ -5451,6 +5606,16 @@ ], "license": "CC-BY-4.0" }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -6701,6 +6866,16 @@ "node": ">=4.0" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -6840,6 +7015,16 @@ "which": "bin/which" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/exponential-backoff": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", @@ -8340,6 +8525,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8361,6 +8547,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8382,6 +8569,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8403,6 +8591,7 @@ "os": [ "freebsd" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8424,6 +8613,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8445,6 +8635,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8466,6 +8657,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8487,6 +8679,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8508,6 +8701,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8529,6 +8723,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -8550,6 +8745,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 12.0.0" }, @@ -9505,6 +9701,20 @@ "node": ">= 0.4" } }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -10759,6 +10969,13 @@ "node": ">=8" } }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/signal-exit": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", @@ -11003,12 +11220,26 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/stats.js": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz", "integrity": "sha512-hNKz8phvYLPEcRkeG1rsGmV5ChMjKDAWU7/OJJdDErPBNChQXxCo3WZurGpnWc6gZhAzEPFad1aVgyOANH1sMw==", "license": "MIT" }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, "node_modules/stream-buffers": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-2.2.0.tgz", @@ -11317,6 +11548,13 @@ "dev": true, "license": "MIT" }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyest": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyest/-/tinyest-0.3.2.tgz", @@ -11339,6 +11577,16 @@ "node": ">=12.20.0" } }, + "node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -11384,6 +11632,16 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -11992,6 +12250,109 @@ } } }, + "node_modules/vitest": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/watchpack": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", @@ -12150,6 +12511,23 @@ "node": ">= 8" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -12407,6 +12785,20 @@ "typescript": "~5.9.3" } }, + "packages/dapi": { + "name": "@diffusionstudio/dapi", + "version": "0.1.0", + "license": "MPL-2.0", + "dependencies": { + "@diffusionstudio/jsx": "*", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "typescript": "~5.9.3", + "vitest": "^4.0.0" + } + }, "packages/encoder": { "name": "@diffusionstudio/encoder", "version": "0.1.0", diff --git a/packages/dapi/README.md b/packages/dapi/README.md new file mode 100644 index 00000000..207f8dd9 --- /dev/null +++ b/packages/dapi/README.md @@ -0,0 +1,15 @@ +# @diffusionstudio/dapi + +The tool catalog behind `dapi`: one entry per tool with its name, description, and zod input and output schemas. The app registers the catalog with its MCP server, the renderer validates incoming calls against it, and the CLI derives its commands and help text from it, so the API is described exactly once. + +```ts +import { catalog, toolByName, Time } from "@diffusionstudio/dapi"; +import type { ToolArgs, ToolResult } from "@diffusionstudio/dapi"; + +const grab = toolByName("media_grab"); +const args: ToolArgs<"media_grab"> = grab.input.parse({ path: "/clip.mp4", times: ["45f", "1:10"] }); +``` + +The main export is free of Node built-ins so the renderer can import it. `@diffusionstudio/dapi/socket` exports the socket path and pulls in `node:os`; only Node processes import it. + +Design: [docs/mcp-server.md](../../docs/mcp-server.md). diff --git a/packages/dapi/package.json b/packages/dapi/package.json new file mode 100644 index 00000000..6a10c94e --- /dev/null +++ b/packages/dapi/package.json @@ -0,0 +1,30 @@ +{ + "name": "@diffusionstudio/dapi", + "version": "0.1.0", + "description": "The dapi tool catalog: every tool the Diffusion Studio app exposes to agents and the CLI, as a name, a description, and zod input/output schemas. The one definition the app serves, the CLI derives its commands from, and the renderer validates against. Depends on zod and the Time parser only; the socket path lives behind the Node-only `./socket` export.", + "license": "MPL-2.0", + "type": "module", + "sideEffects": false, + "exports": { + ".": "./src/index.ts", + "./socket": "./src/socket.ts" + }, + "scripts": { + "check": "tsc --noEmit", + "test": "vitest run" + }, + "keywords": [ + "dapi", + "mcp", + "zod" + ], + "dependencies": { + "@diffusionstudio/jsx": "*", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "typescript": "~5.9.3", + "vitest": "^4.0.0" + } +} diff --git a/packages/dapi/src/catalog.test.ts b/packages/dapi/src/catalog.test.ts new file mode 100644 index 00000000..cba7fd8e --- /dev/null +++ b/packages/dapi/src/catalog.test.ts @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { catalog, isToolName, toolByName } from "./catalog"; + +describe("catalog", () => { + it("has unique MCP-legal names", () => { + const names = catalog.map((tool) => tool.name); + expect(new Set(names).size).toBe(names.length); + for (const name of names) expect(name).toMatch(/^[a-z][a-z0-9_]{0,63}$/); + }); + + it("describes every tool for an agent, not just a label", () => { + for (const tool of catalog) { + expect(tool.title.length, tool.name).toBeGreaterThan(0); + expect(tool.description.length, tool.name).toBeGreaterThan(40); + } + }); + + it("takes an object as every tool's input, as MCP requires", () => { + for (const tool of catalog) { + expect(tool.input, tool.name).toBeInstanceOf(z.ZodObject); + } + }); + + it("converts every schema to JSON Schema without throwing", () => { + for (const tool of catalog) { + expect(() => z.toJSONSchema(tool.input, { io: "input" }), `${tool.name} input`).not.toThrow(); + expect(() => z.toJSONSchema(tool.output), `${tool.name} output`).not.toThrow(); + } + }); + + it("looks tools up by name", () => { + expect(toolByName("capture").runsIn).toBe("renderer"); + expect(toolByName("fonts").runsIn).toBe("main"); + expect(isToolName("media_grab")).toBe(true); + expect(isToolName("media.frame")).toBe(false); + }); +}); diff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.ts new file mode 100644 index 00000000..0db7d724 --- /dev/null +++ b/packages/dapi/src/catalog.ts @@ -0,0 +1,76 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { z } from "zod"; +import type { Tool } from "./tool"; + +import { open } from "./tools/open"; +import { context } from "./tools/context"; +import { capture } from "./tools/capture"; +import { check } from "./tools/check"; +import { exportScene } from "./tools/export"; +import { models } from "./tools/models"; +import { voices } from "./tools/voices"; +import { whoami } from "./tools/whoami"; +import { logs } from "./tools/logs"; +import { screenshot } from "./tools/screenshot"; +import { mediaProbe } from "./tools/media-probe"; +import { mediaGrab } from "./tools/media-grab"; +import { mediaTranscribe } from "./tools/media-transcribe"; +import { mediaFilmstrip } from "./tools/media-filmstrip"; +import { mediaWaveform } from "./tools/media-waveform"; +import { mediaListen } from "./tools/media-listen"; +import { fonts } from "./tools/fonts"; +import { fetchVideo } from "./tools/fetch"; +import { report } from "./tools/report"; + +/** + * Every tool, in the order a listing shows them: the project loop first + * (open, look, capture, check, export), then media inspection, then the + * app and machine utilities. + */ +export const catalog = [ + open, + context, + capture, + check, + exportScene, + mediaProbe, + mediaGrab, + mediaTranscribe, + mediaFilmstrip, + mediaWaveform, + mediaListen, + models, + voices, + whoami, + logs, + screenshot, + fonts, + fetchVideo, + report, +] as const; + +export type AnyTool = (typeof catalog)[number]; +export type ToolName = AnyTool["name"]; +export type ToolByName = Extract; + +/** What a caller sends: times as strings or numbers, defaults omitted. */ +export type ToolInput = z.input["input"]>; +/** What a handler receives: parsed, defaults applied. */ +export type ToolArgs = z.output["input"]>; +export type ToolResult = z.output["output"]>; + +const byName = new Map(catalog.map((tool) => [tool.name, tool])); + +export function toolByName(name: N): ToolByName { + return byName.get(name) as ToolByName; +} + +export function isToolName(name: string): name is ToolName { + return byName.has(name); +} + +/** The catalog as the generic `Tool` shape, for code that iterates it. */ +export const tools: readonly Tool[] = catalog; diff --git a/packages/dapi/src/errors.ts b/packages/dapi/src/errors.ts new file mode 100644 index 00000000..750784a0 --- /dev/null +++ b/packages/dapi/src/errors.ts @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Why a tool call failed, for callers that branch on it: the MCP server maps + * every code to an `isError` result, the CLI maps it to an exit code, and an + * agent reads the message. The message is the part written for people. + */ +export type DapiErrorCode = + /** The tool needs an open project and none is. */ + | "no-project" + /** No node, asset, or file matches the given id or path. */ + | "not-found" + /** The id matches several nodes; the caller must disambiguate. */ + | "ambiguous" + /** The target exists but is the wrong kind, e.g. a clip where a scene is required. */ + | "wrong-kind" + /** The input failed validation beyond what the schema expresses. */ + | "invalid-input" + /** A single-slot resource (the export renderer) is in use. */ + | "busy" + /** The tool needs a signed-in account. */ + | "sign-in-required" + /** The platform or an external binary (yt-dlp, gh, osascript) cannot do it. */ + | "unsupported" + /** The caller or the app canceled the operation. */ + | "canceled"; + +export class DapiError extends Error { + readonly code: DapiErrorCode; + + constructor(code: DapiErrorCode, message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "DapiError"; + this.code = code; + } +} + +export function isDapiError(value: unknown): value is DapiError { + return value instanceof DapiError; +} diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts new file mode 100644 index 00000000..d6ab997b --- /dev/null +++ b/packages/dapi/src/index.ts @@ -0,0 +1,79 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Free of Node built-ins on purpose: the renderer imports this. The socket +// path, which needs node:os, lives in `@diffusionstudio/dapi/socket`. + +import type { z } from "zod"; + +export { defineTool } from "./tool"; +export type { Tool, RunsIn } from "./tool"; + +export { catalog, tools, toolByName, isToolName } from "./catalog"; +export type { AnyTool, ToolName, ToolByName, ToolInput, ToolArgs, ToolResult } from "./catalog"; + +export { Time, NonNegativeTime, TIME_FORMS } from "./time"; +export type { TimeInput } from "./time"; + +export { DapiError, isDapiError } from "./errors"; +export type { DapiErrorCode } from "./errors"; + +export { MAX_FRAMES_PER_SHEET } from "./schemas"; +export { FRAME_CAP } from "./tools/media-grab"; +export { ISSUE_LOG_TAIL } from "./tools/report"; + +// Named request and result types, for handlers that spell out their +// signature. Each is the parsed (output) side of the tool's schema. +import type { LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas"; +import type { GenerationRow as GenerationRowType } from "./tools/context"; +import type { CheckIssue as CheckIssueSchema, CheckIssueCode as CheckIssueCodeSchema } from "./tools/check"; +import type { ExportFormat as ExportFormatSchema, ExportSettings as ExportSettingsSchema } from "./tools/export"; +import type { ModelInfo as ModelInfoSchema } from "./tools/models"; +import type { VoiceInfo as VoiceInfoSchema } from "./tools/voices"; +import type { FrameQuality as FrameQualitySchema } from "./tools/media-grab"; +import type { TranscriptSegment as TranscriptSegmentSchema, TranscriptWord as TranscriptWordSchema } from "./tools/media-transcribe"; +import type { FontFamily as FontFamilySchema } from "./tools/fonts"; +import type { ToolArgs, ToolResult } from "./catalog"; + +export type LogLevel = z.output; +export type LogEntry = z.output; +export type TimecodedImage = z.output; +export type GenerationRow = GenerationRowType; +export type CheckIssueCode = z.output; +export type CheckIssue = z.output; +export type ExportFormat = z.output; +export type ExportSettings = z.output; +export type ModelInfo = z.output; +export type VoiceInfo = z.output; +export type FrameQuality = z.output; +export type TranscriptWord = z.output; +export type TranscriptSegment = z.output; +export type FontFamily = z.output; + +export type OpenRequest = ToolArgs<"open">; +export type OpenResult = ToolResult<"open">; +export type ContextResult = ToolResult<"context">; +export type CaptureRequest = ToolArgs<"capture">; +export type CaptureResult = ToolResult<"capture">; +export type CheckRequest = ToolArgs<"check">; +export type CheckResult = ToolResult<"check">; +export type ExportRequest = ToolArgs<"export">; +export type ExportResult = ToolResult<"export">; +export type ModelsRequest = ToolArgs<"models">; +export type LogsRequest = ToolArgs<"logs">; +export type ScreenshotResult = ToolResult<"screenshot">; +export type MediaProbeRequest = ToolArgs<"media_probe">; +export type MediaFrameRequest = ToolArgs<"media_grab">; +export type MediaFrameResult = ToolResult<"media_grab">; +export type MediaTranscribeRequest = ToolArgs<"media_transcribe">; +export type MediaTranscribeResult = ToolResult<"media_transcribe">; +export type MediaFilmstripRequest = ToolArgs<"media_filmstrip">; +export type MediaFilmstripResult = ToolResult<"media_filmstrip">; +export type MediaWaveformRequest = ToolArgs<"media_waveform">; +export type MediaWaveformResult = ToolResult<"media_waveform">; +export type MediaListenRequest = ToolArgs<"media_listen">; +export type MediaListenResult = ToolResult<"media_listen">; +export type FontsRequest = ToolArgs<"fonts">; +export type FetchRequest = ToolArgs<"fetch">; +export type ReportRequest = ToolArgs<"report">; diff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.ts new file mode 100644 index 00000000..22ffad93 --- /dev/null +++ b/packages/dapi/src/schemas.ts @@ -0,0 +1,97 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Building blocks several tools share. A tool file composes these; nothing +// here is a tool on its own. + +import { z } from "zod"; +import { NonNegativeTime } from "./time"; + +export const NodeId = z + .string() + .min(1) + .describe("node id from the project's JSX, or `file:id` when two files use the same id"); + +export const SceneId = z + .string() + .min(1) + .describe("scene id from the project's JSX, or `file:id` when two files use the same id"); + +export const AssetPath = z + .string() + .min(1) + .describe( + "absolute file path or URL (works with or without an open project), or a library path like `b-roll/clip.mp4` (needs an open project)", + ); + +/** Beyond this the cells get too small to be worth the tokens; use `media_filmstrip`. */ +export const MAX_FRAMES_PER_SHEET = 12; + +/** + * One written image: a single frame stamped with its timecode, or a contact + * sheet stamped with the span it covers (`0f-08s10f`). + */ +export const TimecodedImage = z.object({ + timecode: z.string(), + base64: z.string().describe("PNG bytes, base64"), +}); + +/** + * How frames are laid out: merged into contact sheets (the default) or one + * image each. `perSheet` only means something for sheets; `checkSheetOptions` + * rejects it otherwise, so add it to any object that spreads these fields. + */ +export const sheetFields = { + combine: z + .boolean() + .default(true) + .describe("merge the images into contact sheets of up to 12 cells, each labelled with its timecode (default); false writes one image per position"), + perSheet: z + .int() + .min(1) + .max(MAX_FRAMES_PER_SHEET) + .optional() + .describe("positions per contact sheet, 1-12; fewer means a larger cell each (default: as many as fit)"), +}; + +export function checkSheetOptions( + value: { combine: boolean; perSheet?: number | undefined }, + ctx: z.RefinementCtx, +): void { + if (value.perSheet !== undefined && !value.combine) { + ctx.addIssue({ + code: "custom", + path: ["perSheet"], + message: "perSheet lays out contact sheets; it cannot be combined with combine: false", + }); + } +} + +/** A `[start, end)` window in seconds; both optional, and start must precede end. */ +export const windowFields = { + start: NonNegativeTime.optional(), + end: NonNegativeTime.optional(), +}; + +export function checkWindow( + value: { start?: number | undefined; end?: number | undefined }, + ctx: z.RefinementCtx, +): void { + if (value.start !== undefined && value.end !== undefined && value.start >= value.end) { + ctx.addIssue({ + code: "custom", + path: ["end"], + message: `start (${value.start}s) must be less than end (${value.end}s)`, + }); + } +} + +export const LogLevel = z.enum(["debug", "info", "warning", "error"]); + +export const LogEntry = z.object({ + ts: z.number().describe("unix time, milliseconds"), + level: LogLevel, + message: z.string(), + source: z.string().describe("file:line the entry came from, or empty"), +}); diff --git a/apps/cli/src/cli-socket-path.ts b/packages/dapi/src/socket.ts similarity index 80% rename from apps/cli/src/cli-socket-path.ts rename to packages/dapi/src/socket.ts index c0e535e6..d38ad1c9 100644 --- a/apps/cli/src/cli-socket-path.ts +++ b/packages/dapi/src/socket.ts @@ -8,8 +8,8 @@ import { join } from "node:path"; // One socket / named pipe per host. On macOS tmpdir is per-user; on Linux /tmp // is global but the socket file's owner-only mode 0600 keeps it isolated. // -// Kept separate from cli-channels so the renderer can import the channel -// registry and envelope types without pulling in node:os / node:path. +// Its own entry point (`@diffusionstudio/dapi/socket`) so the renderer can +// import the catalog without pulling in node:os / node:path. export const SOCKET_PATH = platform() === "win32" ? "\\\\.\\pipe\\diffusion-studio" diff --git a/packages/dapi/src/time.test.ts b/packages/dapi/src/time.test.ts new file mode 100644 index 00000000..c3c299e9 --- /dev/null +++ b/packages/dapi/src/time.test.ts @@ -0,0 +1,52 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from "vitest"; +import { NonNegativeTime, Time } from "./time"; + +describe("Time", () => { + it.each([ + [1.5, 1.5], + ["1.5", 1.5], + ["45f", 1.5], + ["-30f", -1], + ["1:30", 90], + ["01:02:03", 3723], + ["-1", -1], + [" 2 ", 2], + ])("parses %j to %d seconds", (input, seconds) => { + expect(Time.parse(input)).toBe(seconds); + }); + + it.each(["", "abc", "1:2:3:4", "1:x"])( + "rejects %j with a message that names the accepted forms", + (input) => { + const result = Time.safeParse(input); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]!.message).toMatch(/seconds \("1.5"\), frames \("45f"\), or "MM:SS"/); + }, + ); + + it("rejects other JSON types before trying to parse them", () => { + expect(Time.safeParse(null).success).toBe(false); + expect(Time.safeParse({ seconds: 1 }).success).toBe(false); + expect(Time.safeParse(Number.NaN).success).toBe(false); + expect(Time.safeParse(Number.POSITIVE_INFINITY).success).toBe(false); + }); +}); + +describe("NonNegativeTime", () => { + it("accepts zero and positive times", () => { + expect(NonNegativeTime.parse(0)).toBe(0); + expect(NonNegativeTime.parse("0:10")).toBe(10); + }); + + it("rejects negative times after parsing them", () => { + const result = NonNegativeTime.safeParse("-1f"); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]!.message).toMatch(/non-negative/); + }); +}); diff --git a/packages/dapi/src/time.ts b/packages/dapi/src/time.ts new file mode 100644 index 00000000..31a6c5a0 --- /dev/null +++ b/packages/dapi/src/time.ts @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { parseTime } from "@diffusionstudio/jsx"; + +export const TIME_FORMS = `seconds ("1.5"), frames ("45f"), or "MM:SS"`; + +/** + * A point in time as agents and the CLI write it — a number of seconds, a + * frame count like "45f", or a clock string like "1:30" — parsed to seconds. + * Negative values are allowed; wrap with `nonNegative` where they are not. + */ +export const Time = z + .union([z.number(), z.string()]) + .transform((value, ctx) => { + const seconds = parseTime(value); + if (seconds === undefined) { + ctx.addIssue({ code: "custom", message: `expected a time — ${TIME_FORMS} (got "${value}")` }); + return z.NEVER; + } + return seconds; + }); + +export const NonNegativeTime = Time.refine((seconds) => seconds >= 0, { + error: `expected a non-negative time — ${TIME_FORMS}`, +}); + +export type TimeInput = z.input; diff --git a/packages/dapi/src/tool.ts b/packages/dapi/src/tool.ts new file mode 100644 index 00000000..7c4f1818 --- /dev/null +++ b/packages/dapi/src/tool.ts @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { z } from "zod"; + +/** + * Which process answers the tool. Renderer tools need the open project's + * world, the engine, or browser media APIs; main tools need the file system + * or a child process and run without a window. + */ +export type RunsIn = "renderer" | "main"; + +export interface Tool< + Name extends string = string, + Input extends z.ZodObject = z.ZodObject, + Output extends z.ZodType = z.ZodType, +> { + /** MCP tool name: `[a-z0-9_]`, unique across the catalog. */ + readonly name: Name; + /** Short human label, a few words. */ + readonly title: string; + /** What the tool does, for an agent choosing between tools. */ + readonly description: string; + /** Always an object: MCP tool arguments are a JSON object by definition. */ + readonly input: Input; + readonly output: Output; + readonly runsIn: RunsIn; +} + +/** Identity with inference: keeps the literal name and the exact schema types. */ +export function defineTool( + tool: Tool, +): Tool { + return tool; +} diff --git a/packages/dapi/src/tools/capture.ts b/packages/dapi/src/tools/capture.ts new file mode 100644 index 00000000..56328380 --- /dev/null +++ b/packages/dapi/src/tools/capture.ts @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { checkSheetOptions, SceneId, sheetFields, TimecodedImage } from "../schemas"; + +export const capture = defineTool({ + name: "capture", + title: "Capture frames", + description: + "Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per position, at 720p height). The tool for checking composition (\"what plays at time T\": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use media_grab.", + input: z + .object({ + id: SceneId, + frames: z + .array(z.int().nonnegative()) + .optional() + .describe("positions to capture as frame numbers relative to the export's first frame, the workarea's start (default: [0])"), + ...sheetFields, + }) + .superRefine(checkSheetOptions), + output: z.array(TimecodedImage), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/check.ts b/packages/dapi/src/tools/check.ts new file mode 100644 index 00000000..74fe81d6 --- /dev/null +++ b/packages/dapi/src/tools/check.ts @@ -0,0 +1,49 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { NodeId } from "../schemas"; + +export const CheckIssueCode = z.enum([ + "black-frames", + "no-visuals", + "never-visible", + "zero-duration", + "transparent", + "source-error", +]); + +/** + * One structural finding. `ranges` (where present) are seconds relative to + * the checked node's start — the same clock `capture` positions use. + */ +export const CheckIssue = z.object({ + code: CheckIssueCode, + severity: z.enum(["error", "warning"]), + message: z.string(), + node: z + .string() + .optional() + .describe("source stamp of the offending node; absent when the issue is about the subtree as a whole"), + ranges: z.array(z.object({ start: z.number(), end: z.number() })).optional(), +}); + +export const check = defineTool({ + name: "check", + title: "Check structure", + description: + "Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load or generate — plus subtree stats (node count by kind, nesting depth, played duration). Times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock capture uses. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with capture.", + input: z.object({ id: NodeId }), + output: z.object({ + stats: z.object({ + nodes: z.int().describe("nodes in the subtree, the checked node included"), + byKind: z.record(z.string(), z.int()), + depth: z.int().describe("deepest nesting level below the checked node (0 = no children)"), + duration: z.number().describe("seconds the checked node plays (its workarea, when one is set)"), + }), + issues: z.array(CheckIssue), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.ts new file mode 100644 index 00000000..c6b40ba8 --- /dev/null +++ b/packages/dapi/src/tools/context.ts @@ -0,0 +1,54 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +const GenerationRow = z.object({ + element: z + .string() + .nullable() + .describe("the element's source stamp, `:`; null for an entity no element produced"), + name: z.string().nullable(), + state: z.enum(["generating", "failed", "done"]), + error: z.string().optional().describe("what the generation failed with, on failed rows"), + asset: z + .string() + .optional() + .describe("the library path the generation landed as, on done rows — ready for media_probe and its siblings"), +}); + +/** + * What the project's source cannot say: the JSX already holds the scenes, + * the selection, and the work area, so the report is only the folders, the + * playhead, the fonts actually registered, and where generations stand. + */ +export const context = defineTool({ + name: "context", + title: "App context", + description: + "Report the current app context: the application root folder (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and where its `generate.*` declarations stand. Poll it to wait for generations without blocking.", + input: z.object({}), + output: z.union([ + z.object({ + rootDir: z.string().describe("folder projects live under"), + projectDir: z.null(), + }), + z.object({ + rootDir: z.string().describe("folder projects live under"), + projectDir: z.string().describe("absolute path of the open project"), + currentTime: z + .number() + .nullable() + .describe("playhead in seconds, the unit the source places clips in; null when no scene is active"), + fontFamilies: z + .array(z.string()) + .describe("families registered in the world drawing the project; the editor default is always among them"), + generations: z.array(GenerationRow), + }), + ]), + runsIn: "renderer", +}); + +export type GenerationRow = z.output; diff --git a/packages/dapi/src/tools/export.ts b/packages/dapi/src/tools/export.ts new file mode 100644 index 00000000..cf5cfc72 --- /dev/null +++ b/packages/dapi/src/tools/export.ts @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { SceneId } from "../schemas"; + +export const ExportFormat = z.enum(["mp4", "webm", "ogg", "mov"]); + +// The settings shape mirrors the scene's `diffusion.export.` entry in the +// project's package.json (see the web app's engine/project-config). Codecs +// are strings on the wire; the app validates them against what the encoder +// accepts. +export const ExportSettings = z.object({ + format: ExportFormat.optional(), + video: z + .object({ + enabled: z.boolean().optional(), + codec: z.string().optional(), + bitrate: z.number().optional(), + fps: z.number().optional(), + resolution: z.number().optional(), + }) + .optional(), + audio: z + .object({ + enabled: z.boolean().optional(), + codec: z.string().optional(), + sampleRate: z.number().optional(), + bitrate: z.number().optional(), + }) + .optional(), +}); + +export const exportScene = defineTool({ + name: "export", + title: "Export scene", + description: + "Encode a scene to a video file — the same render the app's export runs, covering the scene's workarea. Settings come from the scene's `diffusion.export.` entry in the project's package.json (the entry the app's export panel writes); a scene without one exports with the defaults (1080p H.264 MP4, AAC audio). The output path's extension picks the container, overriding the configured format. Returns the written path and the settings used. One export runs at a time; progress shows in the app. Only export when asked to: capture is the tool for checking a composition.", + input: z.object({ + id: SceneId, + path: z + .string() + .optional() + .describe( + "absolute output file path, ffmpeg-style; its extension picks the container (default: exports/. in the project folder)", + ), + }), + output: z.object({ + path: z.string(), + width: z.number().describe("encoded pixel width; 0 for an audio-only export"), + height: z.number().describe("encoded pixel height; 0 for an audio-only export"), + duration: z.number().describe("seconds"), + size: z.number().describe("bytes"), + config: ExportSettings.describe( + "the settings the export was made with — the package.json entry (or the defaults), with the container the extension resolved to", + ), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/fetch.ts b/packages/dapi/src/tools/fetch.ts new file mode 100644 index 00000000..b628c542 --- /dev/null +++ b/packages/dapi/src/tools/fetch.ts @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +export const fetchVideo = defineTool({ + name: "fetch", + title: "Fetch video", + description: + "Download a video with yt-dlp (installed separately). Writes files to disk only and returns their paths (a single URL can yield several, e.g. a playlist).", + input: z.object({ + url: z.string().min(1).describe("video or page URL to download"), + output: z.string().optional().describe("output file path or directory (yt-dlp -o template; default: yt-dlp's default)"), + format: z.string().optional().describe('yt-dlp format selector (default: prefer mp4), e.g. "bv*+ba/b"'), + audio: z.boolean().optional().describe("extract audio only (yt-dlp -x)"), + raw: z.array(z.string()).optional().describe('raw yt-dlp flags passed through, e.g. ["--sponsorblock-remove", "all"]'), + }), + output: z.object({ paths: z.array(z.string()) }), + runsIn: "main", +}); diff --git a/packages/dapi/src/tools/fonts.ts b/packages/dapi/src/tools/fonts.ts new file mode 100644 index 00000000..4936dbfe --- /dev/null +++ b/packages/dapi/src/tools/fonts.ts @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +const FontStyle = z.enum(["normal", "italic"]); + +export const FontFamily = z.object({ + family: z.string(), + variants: z.array( + z.object({ + weight: z.string().describe("CSS weight, 100-900"), + style: FontStyle, + source: z.string().describe("CSS `local()` source list"), + }), + ), +}); + +export const fonts = defineTool({ + name: "fonts", + title: "Local fonts", + description: + "List the local fonts available on this machine (macOS only). These family names are valid `fontFamily` values on ; each family lists its variants.", + input: z.object({ + family: z.string().optional().describe("filter to families whose name contains this (case-insensitive)"), + weights: z.array(z.string()).optional().describe('filter to variants with the given CSS weights, e.g. ["400", "700"]'), + style: FontStyle.optional().describe("filter to variants with the given style"), + limit: z.int().min(1).optional().describe("return at most this many families"), + }), + output: z.array(FontFamily), + runsIn: "main", +}); diff --git a/packages/dapi/src/tools/logs.ts b/packages/dapi/src/tools/logs.ts new file mode 100644 index 00000000..d959265b --- /dev/null +++ b/packages/dapi/src/tools/logs.ts @@ -0,0 +1,20 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { LogEntry, LogLevel } from "../schemas"; + +export const logs = defineTool({ + name: "logs", + title: "App logs", + description: + "Recent console output from the running app (what the devtools console shows: page logs, worker logs, uncaught errors), oldest first. The app buffers the last 2000 entries across reloads and project switches, so this replaces relaunching with ELECTRON_ENABLE_LOGGING=1 when debugging renderer-side behavior.", + input: z.object({ + tail: z.int().min(1).optional().describe("return only the last n entries"), + level: LogLevel.optional().describe("minimum level to include"), + }), + output: z.array(LogEntry), + runsIn: "main", +}); diff --git a/packages/dapi/src/tools/media-filmstrip.ts b/packages/dapi/src/tools/media-filmstrip.ts new file mode 100644 index 00000000..1de24cc2 --- /dev/null +++ b/packages/dapi/src/tools/media-filmstrip.ts @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath, checkWindow, windowFields } from "../schemas"; + +/** The window and scale that filmstrip and waveform share. */ +export const previewFields = { + ...windowFields, + scale: z + .number() + .positive() + .optional() + .describe("scale factor for the thumbnails; smaller fits more rows and columns, larger fits fewer (default: 1)"), +}; + +export const mediaFilmstrip = defineTool({ + name: "media_filmstrip", + title: "Filmstrip preview", + description: + "Render a grid of thumbnails sampled across the timeline to a PNG (local render, no credits), each row stamped with an HH:MM:SS:FF ruler. A fast, token-efficient video track preview; narrow the window to zoom into a region of interest. Video only (use media_waveform for audio).", + input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), + output: z.looseObject({ base64: z.string().describe("PNG bytes, base64") }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/media-grab.ts b/packages/dapi/src/tools/media-grab.ts new file mode 100644 index 00000000..16df6aa2 --- /dev/null +++ b/packages/dapi/src/tools/media-grab.ts @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath, checkSheetOptions, checkWindow, sheetFields, TimecodedImage, windowFields } from "../schemas"; +import { Time } from "../time"; + +export const FrameQuality = z.enum(["small", "medium", "large", "fullres"]); + +/** Guardrail against accidentally decoding a huge number of frames; `uncapped` lifts it. */ +export const FRAME_CAP = 100; + +export const mediaGrab = defineTool({ + name: "media_grab", + title: "Grab frames", + description: + "Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (`08s10f`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (combine: false writes a PNG per frame). Grabs the asset's own pixels, unlike capture which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer media_filmstrip.", + input: z + .object({ + path: AssetPath, + times: z + .array(Time) + .optional() + .describe( + 'timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: [0])', + ), + count: z + .int() + .min(1) + .optional() + .describe("instead of times, grab this many frames evenly spaced across the clip (or across the start/end window)"), + auto: z + .boolean() + .optional() + .describe( + "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU", + ), + ...windowFields, + quality: FrameQuality.optional().describe( + "frame resolution: small (384x384), medium (768x768), large (1536x1536), or fullres (native); default: as large as the sheet cell allows, or small with combine: false", + ), + ...sheetFields, + uncapped: z + .boolean() + .optional() + .describe(`lift the ${FRAME_CAP}-frame safety cap (grabbing many frames is slow and token-heavy)`), + }) + .superRefine((value, ctx) => { + if (value.times !== undefined && value.count !== undefined) { + ctx.addIssue({ code: "custom", path: ["count"], message: "pass either times or count, not both" }); + } + if (value.auto && value.times !== undefined) { + ctx.addIssue({ code: "custom", path: ["times"], message: "auto picks its own timestamps; it cannot be combined with times" }); + } + const windowed = value.start !== undefined || value.end !== undefined; + if (windowed && value.count === undefined && !value.auto) { + ctx.addIssue({ code: "custom", path: ["start"], message: "start and end only apply together with count or auto" }); + } + checkWindow(value, ctx); + const requested = value.count ?? value.times?.length ?? 1; + if (!value.uncapped && requested > FRAME_CAP) { + ctx.addIssue({ + code: "custom", + path: [value.count !== undefined ? "count" : "times"], + message: `grabbing ${requested} frames exceeds the ${FRAME_CAP}-frame cap; pass uncapped: true to override`, + }); + } + checkSheetOptions(value, ctx); + }), + output: z.array(TimecodedImage), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/media-listen.ts b/packages/dapi/src/tools/media-listen.ts new file mode 100644 index 00000000..c1d13e4d --- /dev/null +++ b/packages/dapi/src/tools/media-listen.ts @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath, checkWindow, windowFields } from "../schemas"; + +export const mediaListen = defineTool({ + name: "media_listen", + title: "Listen to audio", + description: + "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; by default only the audio track is analyzed. Needs a signed-in account.", + input: z + .object({ + path: AssetPath, + prompt: z.string().optional().describe("question or instruction to guide the analysis"), + start: windowFields.start.describe( + "start of the segment to analyze (default: 0); timestamps in the analysis are relative to this point", + ), + end: windowFields.end.describe("end of the segment to analyze (default: media duration)"), + stripVideo: z + .boolean() + .optional() + .describe( + "for a video asset, analyze the audio track only (default); false keeps the video so the model also reads what is on screen (expensive: uploads the full video)", + ), + }) + .superRefine(checkWindow), + output: z.object({ + result: z.string().optional(), + start: z.number().optional(), + end: z.number().optional(), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/media-probe.ts b/packages/dapi/src/tools/media-probe.ts new file mode 100644 index 00000000..af6d0eec --- /dev/null +++ b/packages/dapi/src/tools/media-probe.ts @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath } from "../schemas"; + +const Track = z.looseObject({ + id: z.number(), + type: z.string(), + codec: z.string().nullable(), + language: z.string(), + firstTimestamp: z.number(), + duration: z.number(), +}); + +export const mediaProbe = defineTool({ + name: "media_probe", + title: "Probe media", + description: + "Read the container and per-track technical metadata of a media file (local read, no credits): container format, duration, tags, and each track's codec params, without decoding. Commonly useful for a quick technical read, e.g. checking codec compatibility or duration before cutting. Packet stats (fps, bitrate) are estimated from a leading sample; images and transcripts report file-level info only.", + input: z.object({ path: AssetPath }), + output: z.looseObject({ + id: z.string(), + name: z.string(), + path: z.string(), + type: z.string(), + mimeType: z.string().optional(), + size: z.number().describe("bytes"), + width: z.number().optional(), + height: z.number().optional(), + format: z.string().nullable().describe("container name; null when the file could not be read as media"), + duration: z.number().optional().describe("seconds"), + tags: z.record(z.string(), z.unknown()).optional(), + tracks: z.array(Track), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/media-transcribe.ts b/packages/dapi/src/tools/media-transcribe.ts new file mode 100644 index 00000000..c80917ea --- /dev/null +++ b/packages/dapi/src/tools/media-transcribe.ts @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath } from "../schemas"; + +export const TranscriptWord = z.object({ + text: z.string(), + start: z.number().describe("seconds"), + end: z.number().describe("seconds"), +}); + +export const TranscriptSegment = z.object({ + text: z.string(), + words: z.array(TranscriptWord), +}); + +export const mediaTranscribe = defineTool({ + name: "media_transcribe", + title: "Transcribe speech", + description: + "Transcribe the speech in a video or audio file and return the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).", + input: z.object({ path: AssetPath }), + output: z.object({ segments: z.array(TranscriptSegment) }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/media-waveform.ts b/packages/dapi/src/tools/media-waveform.ts new file mode 100644 index 00000000..61a17c45 --- /dev/null +++ b/packages/dapi/src/tools/media-waveform.ts @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; +import { AssetPath, checkWindow } from "../schemas"; +import { previewFields } from "./media-filmstrip"; + +export const mediaWaveform = defineTool({ + name: "media_waveform", + title: "Waveform preview", + description: + "Render the audio track of a video or audio file as a waveform PNG (local render, no credits) with a timestamp ruler: loudness over time, with silent stretches highlighted in red. A fast, token-efficient audio track preview; the silent spans are also returned as second ranges.", + input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), + output: z.looseObject({ + base64: z.string().describe("PNG bytes, base64"), + silences: z.array(z.object({ start: z.number(), end: z.number() })).describe("seconds"), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/models.ts b/packages/dapi/src/tools/models.ts new file mode 100644 index 00000000..f4dade8e --- /dev/null +++ b/packages/dapi/src/tools/models.ts @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +const ModelType = z.enum(["image", "video", "audio"]); + +export const ModelInfo = z.object({ + type: ModelType, + id: z.string(), + name: z.string(), + durations: z.array(z.string()).optional(), + aspectRatios: z.array(z.string()).optional(), + features: z.array(z.enum(["start-frame", "end-frame", "audio"])).optional(), +}); + +export const models = defineTool({ + name: "models", + title: "Generation models", + description: + "List available AI generation models and their per-model constraints (durations, aspect ratios, features), for `generate.*` asset declarations in a project module.", + input: z.object({ + type: ModelType.optional().describe("filter to one kind of model (default: all three)"), + }), + output: z.array(ModelInfo), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/open.ts b/packages/dapi/src/tools/open.ts new file mode 100644 index 00000000..523fbba3 --- /dev/null +++ b/packages/dapi/src/tools/open.ts @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +export const open = defineTool({ + name: "open", + title: "Open project", + description: + "Open a folder as a project in the running app, creating the project files if the folder is not one yet, and show it in the editor. Returns the project's id, display name, and folder. Run this once before tools that need an open project (capture, check, export, context, and library paths in media tools).", + input: z.object({ + dir: z.string().min(1).describe("absolute path of the project folder to open or create"), + }), + output: z.object({ + id: z.string().describe("package.json projectId; empty for a folder that predates ids"), + name: z.string().describe("display name"), + dir: z.string().describe("absolute path of the project folder"), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/report.ts b/packages/dapi/src/tools/report.ts new file mode 100644 index 00000000..6d0523c7 --- /dev/null +++ b/packages/dapi/src/tools/report.ts @@ -0,0 +1,24 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +/** Trailing app log entries attached to an issue by default. */ +export const ISSUE_LOG_TAIL = 50; + +export const report = defineTool({ + name: "report", + title: "Report a bug", + description: + "Report a bug in dapi or the app itself. Files a GitHub issue on diffusionstudio/editor with diagnostics attached (dapi version, platform, recent app logs) and returns its URL. Submits immediately and publicly through the gh CLI, which must be installed and authenticated; there is no review step, so only report real defects and check the attached logs for anything private.", + input: z.object({ + title: z.string().min(1).describe("one-line summary of the problem"), + body: z.string().optional().describe("what happened, in markdown: expected vs actual, and anything the diagnostics won't show"), + commands: z.array(z.string()).optional().describe("the dapi commands or tool calls that reproduce it, in order"), + logs: z.int().min(0).optional().describe(`trailing app log entries to attach (0 to omit; default: ${ISSUE_LOG_TAIL})`), + }), + output: z.object({ url: z.string() }), + runsIn: "main", +}); diff --git a/packages/dapi/src/tools/screenshot.ts b/packages/dapi/src/tools/screenshot.ts new file mode 100644 index 00000000..c5f7b5ed --- /dev/null +++ b/packages/dapi/src/tools/screenshot.ts @@ -0,0 +1,20 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +export const screenshot = defineTool({ + name: "screenshot", + title: "Window screenshot", + description: + "Capture the entire application window as a PNG — the full UI as the user sees it (panels, timeline, asset library, canvas viewport), at the window's current size. The tool for checking what the app itself looks like; to render a node or scene cleanly for composition checks use capture instead.", + input: z.object({}), + output: z.object({ + base64: z.string().describe("PNG bytes, base64"), + width: z.number(), + height: z.number(), + }), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts new file mode 100644 index 00000000..31aa8cb7 --- /dev/null +++ b/packages/dapi/src/tools/tools.test.ts @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from "vitest"; +import { capture } from "./capture"; +import { context } from "./context"; +import { exportScene } from "./export"; +import { logs } from "./logs"; +import { mediaFilmstrip } from "./media-filmstrip"; +import { mediaGrab } from "./media-grab"; +import { mediaListen } from "./media-listen"; + +/** The messages of a failed parse, keyed by the path they point at. */ +function issues(result: { success: boolean; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } }) { + expect(result.success).toBe(false); + return Object.fromEntries((result.error?.issues ?? []).map((issue) => [issue.path.join("."), issue.message])); +} + +describe("media_grab", () => { + const input = mediaGrab.input; + + it("parses times in every form and applies the sheet default", () => { + const args = input.parse({ path: "/clip.mp4", times: ["45f", "1:10", -1, "-2f"] }); + expect(args.times).toEqual([1.5, 70, -1, -2 / 30]); + expect(args.combine).toBe(true); + expect(args.perSheet).toBeUndefined(); + }); + + it("rejects times together with count", () => { + expect(issues(input.safeParse({ path: "/c.mp4", times: [1], count: 3 }))).toHaveProperty("count"); + }); + + it("rejects auto together with times", () => { + expect(issues(input.safeParse({ path: "/c.mp4", times: [1], auto: true }))).toHaveProperty("times"); + }); + + it("requires count or auto for a window", () => { + expect(issues(input.safeParse({ path: "/c.mp4", start: 1 }))).toHaveProperty("start"); + expect(input.safeParse({ path: "/c.mp4", start: 1, count: 2 }).success).toBe(true); + expect(input.safeParse({ path: "/c.mp4", end: "0:10", auto: true }).success).toBe(true); + }); + + it("requires start before end", () => { + const found = issues(input.safeParse({ path: "/c.mp4", start: 5, end: "2", count: 2 })); + expect(found.end).toMatch(/start \(5s\) must be less than end \(2s\)/); + }); + + it("rejects negative window bounds but not negative times", () => { + expect(issues(input.safeParse({ path: "/c.mp4", start: -1, count: 2 }))).toHaveProperty("start"); + expect(input.safeParse({ path: "/c.mp4", times: [-1] }).success).toBe(true); + }); + + it("caps the frame count unless uncapped", () => { + expect(issues(input.safeParse({ path: "/c.mp4", count: 101 })).count).toMatch(/100-frame cap/); + expect(input.safeParse({ path: "/c.mp4", count: 101, uncapped: true }).success).toBe(true); + expect(input.safeParse({ path: "/c.mp4", count: 100 }).success).toBe(true); + }); + + it("rejects perSheet for separate images", () => { + expect(issues(input.safeParse({ path: "/c.mp4", combine: false, perSheet: 4 }))).toHaveProperty("perSheet"); + expect(issues(input.safeParse({ path: "/c.mp4", perSheet: 13 }))).toHaveProperty("perSheet"); + expect(input.safeParse({ path: "/c.mp4", perSheet: 12 }).success).toBe(true); + }); +}); + +describe("capture", () => { + it("takes non-negative integer frames", () => { + expect(capture.input.parse({ id: "intro", frames: [0, 45] }).frames).toEqual([0, 45]); + expect(capture.input.safeParse({ id: "intro", frames: [1.5] }).success).toBe(false); + expect(capture.input.safeParse({ id: "intro", frames: [-1] }).success).toBe(false); + }); + + it("shares the sheet rule with media_grab", () => { + expect(issues(capture.input.safeParse({ id: "intro", combine: false, perSheet: 2 }))).toHaveProperty("perSheet"); + }); +}); + +describe("media_filmstrip and media_listen", () => { + it("apply the window rule", () => { + expect(issues(mediaFilmstrip.input.safeParse({ path: "/c.mp4", start: 3, end: 3 }))).toHaveProperty("end"); + expect(issues(mediaListen.input.safeParse({ path: "/c.mp4", start: "0:05", end: 4 }))).toHaveProperty("end"); + expect(mediaFilmstrip.input.safeParse({ path: "/c.mp4", scale: 0 }).success).toBe(false); + }); +}); + +describe("logs and export", () => { + it("validate the small things the CLI used to check by hand", () => { + expect(logs.input.safeParse({ tail: 0 }).success).toBe(false); + expect(logs.input.safeParse({ tail: 5, level: "warning" }).success).toBe(true); + expect(logs.input.safeParse({ level: "verbose" }).success).toBe(false); + expect(exportScene.input.safeParse({ id: "" }).success).toBe(false); + }); +}); + +describe("context", () => { + it("accepts both the closed and the open report", () => { + expect(context.output.safeParse({ rootDir: "/p", projectDir: null }).success).toBe(true); + expect( + context.output.safeParse({ + rootDir: "/p", + projectDir: "/p/a", + currentTime: null, + fontFamilies: ["Inter"], + generations: [{ element: "index.tsx:3", name: null, state: "done", asset: "gen/a.mp4" }], + }).success, + ).toBe(true); + expect(context.output.safeParse({ rootDir: "/p", projectDir: "/p/a" }).success).toBe(false); + }); +}); diff --git a/packages/dapi/src/tools/voices.ts b/packages/dapi/src/tools/voices.ts new file mode 100644 index 00000000..233cc66a --- /dev/null +++ b/packages/dapi/src/tools/voices.ts @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +export const VoiceInfo = z.object({ + id: z.string(), + label: z.string(), + description: z.string(), +}); + +export const voices = defineTool({ + name: "voices", + title: "Speech voices", + description: "List the speech voices available for `generate.voice` declarations in a project module.", + input: z.object({}), + output: z.array(VoiceInfo), + runsIn: "renderer", +}); diff --git a/packages/dapi/src/tools/whoami.ts b/packages/dapi/src/tools/whoami.ts new file mode 100644 index 00000000..d1ba3f9d --- /dev/null +++ b/packages/dapi/src/tools/whoami.ts @@ -0,0 +1,15 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { z } from "zod"; +import { defineTool } from "../tool"; + +export const whoami = defineTool({ + name: "whoami", + title: "Signed-in account", + description: "Report the authenticated account, or null if signed out.", + input: z.object({}), + output: z.looseObject({ id: z.string(), email: z.string().optional() }).nullable(), + runsIn: "renderer", +}); diff --git a/packages/dapi/tsconfig.json b/packages/dapi/tsconfig.json new file mode 100644 index 00000000..6f6d8c8b --- /dev/null +++ b/packages/dapi/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo", + "lib": ["es2024"], + "types": ["node"] + }, + "include": ["src"] +} From 905d6491412fd10180a23168aa8b3cc3e8620e34 Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Thu, 3 Sep 2026 10:04:02 +0200 Subject: [PATCH 02/13] Refactor CLI and DAPI integration - Removed @trpc/client and @trpc/server dependencies from package-lock.json and apps/cli/package.json. - Updated TypeScript configuration in apps/cli/tsconfig.json to simplify compiler options. - Refactored cli-client.ts to replace TRPC client calls with direct calls to the DAPI, enhancing modularity and performance. - Adjusted media handling functions in index.ts to utilize the new call method for better clarity and maintainability. - Cleaned up protocol.ts by introducing encoding and decoding functions for replies, improving data handling consistency. --- apps/cli/package.json | 2 - apps/cli/src/cli-client.ts | 74 ++-- apps/cli/src/index.ts | 61 ++- apps/cli/src/protocol.ts | 38 ++ apps/cli/tsconfig.json | 39 +- apps/desktop/src/main.ts | 4 +- apps/web/src/app.tsx | 2 +- .../components/canvas/desktop-app-banner.tsx | 2 +- .../sidebar-left/project-menu/index.tsx | 2 +- .../components/sidebar-left/sidebar-left.tsx | 2 +- apps/web/src/context/dapi/api.tsx | 145 ------- apps/web/src/context/dapi/capture.ts | 102 ----- apps/web/src/context/dapi/context.ts | 99 ----- apps/web/src/context/dapi/export.ts | 141 ------- apps/web/src/context/dapi/logs.ts | 21 - apps/web/src/context/dapi/media.ts | 361 ------------------ apps/web/src/context/dapi/models.ts | 47 --- apps/web/src/context/dapi/window.ts | 19 - apps/web/src/dapi/api.tsx | 94 +++++ apps/web/src/dapi/bridge.ts | 100 +++++ apps/web/src/dapi/handler.ts | 40 ++ apps/web/src/dapi/handlers/capture.ts | 53 +++ .../{context/dapi => dapi/handlers}/check.ts | 14 +- apps/web/src/dapi/handlers/context.ts | 82 ++++ apps/web/src/dapi/handlers/export.ts | 118 ++++++ apps/web/src/dapi/handlers/index.ts | 42 ++ apps/web/src/dapi/handlers/logs.ts | 22 ++ apps/web/src/dapi/handlers/media-filmstrip.ts | 15 + apps/web/src/dapi/handlers/media-grab.ts | 132 +++++++ apps/web/src/dapi/handlers/media-listen.ts | 39 ++ apps/web/src/dapi/handlers/media-probe.ts | 73 ++++ .../web/src/dapi/handlers/media-transcribe.ts | 37 ++ apps/web/src/dapi/handlers/media-waveform.ts | 15 + apps/web/src/dapi/handlers/models.ts | 35 ++ apps/web/src/dapi/handlers/open.ts | 7 + apps/web/src/dapi/handlers/screenshot.ts | 10 + .../{context/dapi => dapi/handlers}/voices.ts | 13 +- apps/web/src/dapi/handlers/whoami.ts | 7 + apps/web/src/{context => }/dapi/index.ts | 2 +- apps/web/src/dapi/lib/assets.ts | 41 ++ .../dapi => dapi/lib}/frame-triage.ts | 0 .../src/{context/dapi => dapi/lib}/nodes.ts | 22 +- apps/web/src/dapi/lib/png.ts | 9 + apps/web/src/dapi/lib/scene.ts | 37 ++ apps/web/src/dapi/lib/sheets.ts | 87 +++++ apps/web/src/{context => }/dapi/session.ts | 14 +- apps/web/src/hooks/use-fullscreen-state.ts | 13 +- apps/web/src/lib/cli-rpc.ts | 45 --- apps/web/src/lib/ipc.ts | 94 +---- apps/web/src/pages/editor.tsx | 2 +- apps/web/src/pages/project.tsx | 2 +- docs/mcp-server.md | 2 +- package-lock.json | 2 - packages/dapi/src/catalog.test.ts | 4 +- packages/dapi/src/index.ts | 3 +- packages/dapi/src/schemas.ts | 12 +- packages/dapi/src/tools/context.ts | 4 +- packages/dapi/src/tools/media-filmstrip.ts | 4 +- packages/dapi/src/tools/media-waveform.ts | 4 +- packages/dapi/src/tools/screenshot.ts | 3 +- packages/dapi/src/tools/whoami.ts | 2 +- packages/dapi/src/validate.test.ts | 28 ++ packages/dapi/src/validate.ts | 25 ++ packages/encoder/src/contact-sheet.ts | 17 +- packages/encoder/src/image-encoder.ts | 21 +- packages/encoder/src/index.ts | 1 + packages/encoder/src/png.ts | 21 + 67 files changed, 1350 insertions(+), 1280 deletions(-) delete mode 100644 apps/web/src/context/dapi/api.tsx delete mode 100644 apps/web/src/context/dapi/capture.ts delete mode 100644 apps/web/src/context/dapi/context.ts delete mode 100644 apps/web/src/context/dapi/export.ts delete mode 100644 apps/web/src/context/dapi/logs.ts delete mode 100644 apps/web/src/context/dapi/media.ts delete mode 100644 apps/web/src/context/dapi/models.ts delete mode 100644 apps/web/src/context/dapi/window.ts create mode 100644 apps/web/src/dapi/api.tsx create mode 100644 apps/web/src/dapi/bridge.ts create mode 100644 apps/web/src/dapi/handler.ts create mode 100644 apps/web/src/dapi/handlers/capture.ts rename apps/web/src/{context/dapi => dapi/handlers}/check.ts (95%) create mode 100644 apps/web/src/dapi/handlers/context.ts create mode 100644 apps/web/src/dapi/handlers/export.ts create mode 100644 apps/web/src/dapi/handlers/index.ts create mode 100644 apps/web/src/dapi/handlers/logs.ts create mode 100644 apps/web/src/dapi/handlers/media-filmstrip.ts create mode 100644 apps/web/src/dapi/handlers/media-grab.ts create mode 100644 apps/web/src/dapi/handlers/media-listen.ts create mode 100644 apps/web/src/dapi/handlers/media-probe.ts create mode 100644 apps/web/src/dapi/handlers/media-transcribe.ts create mode 100644 apps/web/src/dapi/handlers/media-waveform.ts create mode 100644 apps/web/src/dapi/handlers/models.ts create mode 100644 apps/web/src/dapi/handlers/open.ts create mode 100644 apps/web/src/dapi/handlers/screenshot.ts rename apps/web/src/{context/dapi => dapi/handlers}/voices.ts (57%) create mode 100644 apps/web/src/dapi/handlers/whoami.ts rename apps/web/src/{context => }/dapi/index.ts (75%) create mode 100644 apps/web/src/dapi/lib/assets.ts rename apps/web/src/{context/dapi => dapi/lib}/frame-triage.ts (100%) rename apps/web/src/{context/dapi => dapi/lib}/nodes.ts (58%) create mode 100644 apps/web/src/dapi/lib/png.ts create mode 100644 apps/web/src/dapi/lib/scene.ts create mode 100644 apps/web/src/dapi/lib/sheets.ts rename apps/web/src/{context => }/dapi/session.ts (70%) delete mode 100644 apps/web/src/lib/cli-rpc.ts create mode 100644 packages/dapi/src/validate.test.ts create mode 100644 packages/dapi/src/validate.ts create mode 100644 packages/encoder/src/png.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index 76346cd6..20c2302e 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -17,8 +17,6 @@ }, "dependencies": { "@babel/core": "^7.29.7", - "@trpc/client": "^11.18.0", - "@trpc/server": "^11.18.0", "@babel/preset-typescript": "^7.27.1", "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts index 6554862f..0b1a5726 100644 --- a/apps/cli/src/cli-client.ts +++ b/apps/cli/src/cli-client.ts @@ -6,17 +6,36 @@ import { connect } from "node:net"; import { randomBytes } from "node:crypto"; import type { AddressInfo } from "node:net"; import { WebSocketServer } from "ws"; -import { createTRPCClient, TRPCClientError } from "@trpc/client"; -import type { TRPCLink } from "@trpc/client"; -import { observable } from "@trpc/server/observable"; import { SOCKET_PATH } from "@diffusionstudio/dapi/socket"; +import { decodeReply } from "./protocol"; + +import type { ToolInput, ToolName, ToolResult } from "@diffusionstudio/dapi"; import type { CliHandshake, CliHandshakeReply, CliReply, CliRequest } from "./protocol"; -import type { AppRouter } from "../../web/src/context/dapi"; const DEFAULT_TIMEOUT_MS = 60000; export const GENERATE_TIMEOUT_MS = 600000; export const EXPORT_TIMEOUT_MS = 3600000; +export type CallOptions = { timeoutMs?: number }; + +/** + * Calls one tool in the running app. Typed by the catalog: the input is what + * the tool's schema accepts, the result what its handler returns (bytes come + * back as bytes; the wire's base64 is decoded here). + */ +export async function call( + name: N, + input: ToolInput, + options: CallOptions = {}, +): Promise> { + return (await transport({ path: name, input }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS)) as ToolResult; +} + +/** Liveness: answered by the renderer's transport before any handler registers. */ +export async function ping(): Promise { + await transport({ path: "ping", input: undefined }, DEFAULT_TIMEOUT_MS); +} + // Asks the app (via the unix socket) to have the renderer dial our WebSocket // server. Main replies once the connect info has been delivered, so a // rejection here means the app is down or the renderer never became ready. @@ -55,10 +74,12 @@ function requestConnection(handshake: CliHandshake, timeoutMs: number): Promise< }); } +// Each call runs over its own short-lived WebSocket server that the renderer +// dials in to. No cancellation: the CLI process exits when the command settles. async function transport(request: CliRequest, timeoutMs: number): Promise { const token = randomBytes(16).toString("hex"); - // Frame batches and other base64 replies can exceed ws's 100 MiB default, - // so disable the payload cap; the server only lives for one request. + // Frame batches can exceed ws's 100 MiB default, so disable the payload + // cap; the server only lives for one request. const wss = new WebSocketServer({ host: "127.0.0.1", port: 0, maxPayload: 0 }); try { @@ -83,15 +104,13 @@ async function transport(request: CliRequest, timeoutMs: number): Promise { try { - const parsed = JSON.parse(raw.toString()) as CliReply; + const parsed = decodeReply(raw.toString()); settle(() => resolve(parsed)); } catch (e) { settle(() => reject(e instanceof Error ? e : new Error(String(e)))); } }); - ws.on("close", () => - settle(() => reject(new Error("App disconnected before replying"))), - ); + ws.on("close", () => settle(() => reject(new Error("App disconnected before replying")))); ws.on("error", (err) => settle(() => reject(err))); ws.send(JSON.stringify(request)); }); @@ -114,45 +133,22 @@ async function transport(request: CliRequest, timeoutMs: number): Promise = - () => - ({ op }) => - observable((observer) => { - const timeoutMs = - typeof op.context.timeoutMs === "number" ? op.context.timeoutMs : DEFAULT_TIMEOUT_MS; - transport({ path: op.path, input: op.input }, timeoutMs) - .then((data) => { - observer.next({ result: { data } }); - observer.complete(); - }) - .catch((err) => observer.error(TRPCClientError.from(err as Error))); - // No cancellation: the CLI process exits when the command settles. - return () => {}; - }); - -export const editor = createTRPCClient({ links: [cliLink] }); - -// Transport failures surface as TRPCClientError wrapping the socket error; -// unwrap to reach errno codes like ENOENT/ECONNREFUSED. +/** The errno of a transport failure (ENOENT/ECONNREFUSED when the app is down), if any. */ export function errnoCode(e: unknown): string | undefined { - if (!(e instanceof TRPCClientError)) return undefined; - return (e.cause as NodeJS.ErrnoException | undefined)?.code; + return (e as NodeJS.ErrnoException | undefined)?.code; } // Bridges the cold-start gap after launching the app. Main only delivers the -// handshake once the renderer has finished loading, and `ping` is answered -// by the always-mounted app router, so a single round-trip proves the app is -// fully up. The retry loop only handles the brief window before the handshake +// handshake once the renderer has finished loading, and `ping` is answered by +// the renderer's transport, so a single round-trip proves the app is fully +// up. The retry loop only handles the brief window before the handshake // socket itself binds (ENOENT/ECONNREFUSED). export async function waitForCliSocket(timeoutMs = 30000): Promise { const start = Date.now(); let lastError: unknown = null; while (Date.now() - start < timeoutMs) { try { - await editor.ping.query(); + await ping(); return; } catch (e) { lastError = e; diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index e8e74f82..b5160bb3 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -11,16 +11,16 @@ import { dirname, isAbsolute, join, resolve } from "node:path"; import { Command } from "commander"; import { version } from "../../../package.json"; import { parseTime, TIME_FPS } from "@diffusionstudio/jsx"; -import { editor, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, waitForCliSocket } from "./cli-client"; +import { call, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, ping, waitForCliSocket } from "./cli-client"; import { listLocalFonts } from "./fonts"; import { buildIssueBody, createIssue } from "./report"; import { fetchVideo } from "./ytdlp"; -import { MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi"; +import { ISSUE_LOG_TAIL, MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi"; import type { FrameQuality, LogEntry, LogLevel, TimecodedImage } from "@diffusionstudio/dapi"; // Long-running commands (renders, AI generation) override the default 60s. -const GENERATE = { context: { timeoutMs: GENERATE_TIMEOUT_MS } }; -const EXPORT = { context: { timeoutMs: EXPORT_TIMEOUT_MS } }; +const GENERATE = { timeoutMs: GENERATE_TIMEOUT_MS }; +const EXPORT = { timeoutMs: EXPORT_TIMEOUT_MS }; const APP_NAME = "Diffusion Studio"; @@ -107,7 +107,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`); mkdirSync(dir, { recursive: true }); try { - const images = await editor.media.frame.query({ + const images = await call("media_grab", { ...target, times, count, @@ -117,6 +117,7 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { auto: opts.auto, combine: !opts.separate, perSheet, + uncapped: opts.uncapped, }); writeImages(images, dir); } catch (e) { @@ -143,7 +144,7 @@ async function mediaProbe(ref: string): Promise { const target = resolveAssetRef(ref); const stop = startSpinner("Probing asset"); try { - const result = await editor.media.probe.query(target); + const result = await call("media_probe", target); stop(); console.log(JSON.stringify(result)); } catch (e) { @@ -156,7 +157,7 @@ async function mediaTranscribe(ref: string): Promise { const target = resolveAssetRef(ref); const stop = startSpinner("Transcribing asset"); try { - const result = await editor.media.transcribe.query(target, GENERATE); + const result = await call("media_transcribe", target, GENERATE); stop(); console.log(JSON.stringify(result)); } catch (e) { @@ -178,7 +179,8 @@ async function mediaListen(ref: string, opts: MediaListenOptions): Promise const target = resolveAssetRef(ref); const stop = startSpinner("Analyzing asset"); try { - const result = await editor.media.listen.query( + const result = await call( + "media_listen", { ...target, prompt: opts.prompt, start, end, stripVideo: !opts.keepVideo }, GENERATE, ); @@ -207,9 +209,9 @@ function parseTimeArg(value: string, flag: string, allowNegative = false): numbe // image with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the // filename too. function writeImages(images: TimecodedImage[], dir: string): void { - for (const { timecode, base64 } of images) { + for (const { timecode, png } of images) { const path = join(dir, `${timecode}.png`); - writeFileSync(path, Buffer.from(base64, "base64")); + writeFileSync(path, png); console.log(JSON.stringify({ timecode, path })); } } @@ -256,9 +258,9 @@ async function mediaFilmstrip(ref: string, opts: MediaPreviewOptions): Promise { const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`); mkdirSync(dir, { recursive: true }); try { - const images = await editor.capture.query( - { id, frames, combine: !opts.separate, perSheet }, - GENERATE, - ); + const images = await call("capture", { id, frames, combine: !opts.separate, perSheet }, GENERATE); writeImages(images, dir); } catch (e) { handleSocketError(e); @@ -310,7 +309,7 @@ async function exportScene(id: string, output: string | undefined): Promise { try { - const result = await editor.check.query({ id }); + const result = await call("check", { id }); console.log(JSON.stringify(result)); // Linter convention: issues found is a different failure than "could not run". if (result.issues.some((issue) => issue.severity === "error")) process.exitCode = 1; @@ -348,10 +347,10 @@ async function openProject(path: string | undefined, opts: OpenOptions): Promise // A cold launch needs the renderer up before the app can answer; when // nothing was launched there is nothing to wait for, so fail fast. if (launched) await waitForCliSocket(); - else await editor.ping.query(); + else await ping(); if (path !== undefined) { - const result = await editor.open.mutate({ dir: resolve(path) }); + const result = await call("open", { dir: resolve(path) }); console.log(JSON.stringify(result)); } } catch (e) { @@ -361,7 +360,7 @@ async function openProject(path: string | undefined, opts: OpenOptions): Promise async function context(): Promise { try { - const result = await editor.context.query(); + const result = await call("context", {}); console.log(JSON.stringify(result)); } catch (e) { handleSocketError(e); @@ -370,7 +369,7 @@ async function context(): Promise { async function whoami(): Promise { try { - const result = await editor.whoami.query(); + const result = await call("whoami", {}); console.log(JSON.stringify(result)); } catch (e) { handleSocketError(e); @@ -397,7 +396,7 @@ async function showLogs(opts: LogsOptions): Promise { } try { - const entries = await editor.logs.query({ tail, level: opts.level as LogLevel | undefined }); + const entries = await call("logs", { tail, level: opts.level as LogLevel | undefined }); for (const entry of entries) console.log(formatLogEntry(entry)); } catch (e) { handleSocketError(e); @@ -427,14 +426,14 @@ async function appScreenshot(opts: ScreenshotOptions): Promise { const dir = opts.output ?? tmpdir(); mkdirSync(dir, { recursive: true }); try { - const { base64, width, height } = await editor.screenshot.query(); + const { png, width, height } = await call("screenshot", {}); const taken = new Date(); let attempt = 1; let path = join(dir, screenshotFilename(taken, attempt)); while (existsSync(path)) { path = join(dir, screenshotFilename(taken, ++attempt)); } - writeFileSync(path, Buffer.from(base64, "base64")); + writeFileSync(path, png); console.log(JSON.stringify({ path, width, height })); } catch (e) { handleSocketError(e); @@ -443,8 +442,6 @@ async function appScreenshot(opts: ScreenshotOptions): Promise { type IssueOptions = { body?: string; command?: string[]; logs?: string }; -const ISSUE_LOG_TAIL = 50; - async function reportIssue(title: string, opts: IssueOptions): Promise { const summary = title.trim(); if (!summary) { @@ -468,7 +465,7 @@ async function reportIssue(title: string, opts: IssueOptions): Promise { let appStatus = "not checked"; if (tail > 0) { try { - logs = (await editor.logs.query({ tail })).map(formatLogEntry); + logs = (await call("logs", { tail })).map(formatLogEntry); appStatus = "running"; } catch (e) { const code = errnoCode(e); @@ -525,7 +522,7 @@ async function listModels(type: string | undefined): Promise { process.exit(1); } try { - const models = await editor.models.query({ type: type as "image" | "video" | "audio" | undefined }); + const models = await call("models", { type: type as "image" | "video" | "audio" | undefined }); for (const model of models) console.log(JSON.stringify(model)); } catch (e) { handleSocketError(e); @@ -534,7 +531,7 @@ async function listModels(type: string | undefined): Promise { async function listVoices(): Promise { try { - const voices = await editor.voices.query(); + const voices = await call("voices", {}); for (const voice of voices) console.log(JSON.stringify(voice)); } catch (e) { handleSocketError(e); diff --git a/apps/cli/src/protocol.ts b/apps/cli/src/protocol.ts index 9119d703..eda36e18 100644 --- a/apps/cli/src/protocol.ts +++ b/apps/cli/src/protocol.ts @@ -33,3 +33,41 @@ export type CliRequest = { export type CliReply = | { ok: true; data: unknown } | { ok: false; error: string }; + +// Replies carry bytes (PNGs) inside JSON. A Uint8Array is written as +// `{ $bytes: }` and read back as a Uint8Array, so handlers and the +// CLI both see bytes and only this seam knows about base64. Chunked so a +// 100 MiB frame batch never builds a single giant argument list. +const BYTES_KEY = "$bytes"; +const CHUNK = 0x8000; + +export function encodeReply(reply: CliReply): string { + return JSON.stringify(reply, (_key, value) => + value instanceof Uint8Array ? { [BYTES_KEY]: bytesToBase64(value) } : value, + ); +} + +export function decodeReply(text: string): CliReply { + return JSON.parse(text, (_key, value) => + isBytesEnvelope(value) ? base64ToBytes(value[BYTES_KEY]) : value, + ) as CliReply; +} + +function isBytesEnvelope(value: unknown): value is { [BYTES_KEY]: string } { + return typeof value === "object" && value !== null && typeof (value as Record)[BYTES_KEY] === "string"; +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)); + } + return btoa(binary); +} + +function base64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json index c3835eb7..ac304146 100644 --- a/apps/cli/tsconfig.json +++ b/apps/cli/tsconfig.json @@ -2,40 +2,9 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo", - // The AppRouter type import in cli-client.ts pulls renderer sources into - // this program, so the web app's compiler settings (DOM, solid JSX, vite - // types, aliases) apply here on top of the node ones. - "useDefineForClassFields": true, - "erasableSyntaxOnly": false, - "lib": [ - "es2024", - "DOM", - "DOM.Iterable" - ], - "types": [ - "node", - "vite-plugin-solid-svg/types-component-solid", - "vite/client", - "@types/audioworklet", - "@types/wicg-file-system-access", - "@webgpu/types" - ], - "resolveJsonModule": true, - "allowImportingTsExtensions": true, - "jsx": "preserve", - "jsxImportSource": "solid-js", - "baseUrl": "./", - "paths": { - "@/*": [ - "../web/src/*" - ], - "@desktop/*": [ - "../desktop/src/*" - ] - } + "lib": ["es2024"], + "types": ["node"], + "resolveJsonModule": true }, - "include": [ - "src", - "../web/src/vite-env.d.ts" - ] + "include": ["src"] } diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index ad16a026..bd79dd7c 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -275,7 +275,9 @@ if (app.requestSingleInstanceLock()) { if (!mainWindow || mainWindow.isDestroyed()) throw new Error("No main window"); const image = await mainWindow.webContents.capturePage(undefined, { stayHidden: true }); const { width, height } = image.getSize(); - return { base64: image.toPNG().toString("base64"), width, height }; + const png = image.toPNG(); + // A plain Uint8Array over the PNG, so the renderer sees bytes and not a Buffer. + return { png: new Uint8Array(png.buffer, png.byteOffset, png.byteLength), width, height }; }); mainBridge.handle(MAIN_CHANNELS.HEADLESS_GET_MODE, () => isHeadless()); mainBridge.handle(MAIN_CHANNELS.LOGS_GET, () => logBuffer); diff --git a/apps/web/src/app.tsx b/apps/web/src/app.tsx index bb490a29..39361eb9 100644 --- a/apps/web/src/app.tsx +++ b/apps/web/src/app.tsx @@ -10,7 +10,7 @@ import { AppContextMenu } from "@/components/app-context-menu"; import { AuthProvider, useAuth } from '@/context/auth'; import { PersistRoute } from '@/lib/persist-route'; -import { EditorApi } from '@/context/dapi'; +import { EditorApi } from '@/dapi'; import { UpgradeDialog } from '@/components/upgrade-dialog'; import { PurchaseSuccess } from '@/components/purchase-success'; import { ScreenTooSmall } from '@/components/screen-too-small'; diff --git a/apps/web/src/components/canvas/desktop-app-banner.tsx b/apps/web/src/components/canvas/desktop-app-banner.tsx index c1637fc1..5db091c6 100644 --- a/apps/web/src/components/canvas/desktop-app-banner.tsx +++ b/apps/web/src/components/canvas/desktop-app-banner.tsx @@ -5,7 +5,7 @@ import { Show } from "solid-js"; import { Button } from "@/components/ui/button"; import { Icon } from "@/components/ui/icon"; -import { useEditorApi } from "@/context/dapi"; +import { useEditorApi } from "@/dapi"; import { createStoredSignal } from "@/lib/store"; import { downloadDesktopApp } from "@/lib/desktop-app"; import { track } from "@/lib/analytics"; diff --git a/apps/web/src/components/sidebar-left/project-menu/index.tsx b/apps/web/src/components/sidebar-left/project-menu/index.tsx index 2d2735a5..d9838de6 100644 --- a/apps/web/src/components/sidebar-left/project-menu/index.tsx +++ b/apps/web/src/components/sidebar-left/project-menu/index.tsx @@ -19,7 +19,7 @@ import { import { useNavigate } from "@solidjs/router"; import { Show, onCleanup, onMount } from "solid-js"; import { isInputTarget } from "@/utils"; -import { useEditorApi } from "@/context/dapi"; +import { useEditorApi } from "@/dapi"; import { downloadDesktopApp } from "@/lib/desktop-app"; import { FileMenu } from "./file-menu"; import { EditMenu } from "./edit-menu"; diff --git a/apps/web/src/components/sidebar-left/sidebar-left.tsx b/apps/web/src/components/sidebar-left/sidebar-left.tsx index a179fce4..b71c9db2 100644 --- a/apps/web/src/components/sidebar-left/sidebar-left.tsx +++ b/apps/web/src/components/sidebar-left/sidebar-left.tsx @@ -4,7 +4,7 @@ import { Assets } from "./assets"; import { useLayout } from "@/context/layout"; -import { useEditorApi } from "@/context/dapi"; +import { useEditorApi } from "@/dapi"; import { createSignal, Show } from "solid-js"; import { toast } from "somoto"; import { Button } from "../ui/button"; diff --git a/apps/web/src/context/dapi/api.tsx b/apps/web/src/context/dapi/api.tsx deleted file mode 100644 index 1619cf04..00000000 --- a/apps/web/src/context/dapi/api.tsx +++ /dev/null @@ -1,145 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { createEffect, createContext, useContext, onCleanup } from "solid-js"; -import { useNavigate } from "@solidjs/router"; -import { useWorld } from '@diffusionstudio/koota-solid'; -import { Project } from '@diffusionstudio/runtime'; -import { useProject } from '@/context/project'; -import { useAuth } from '@/context/auth'; -import { useEngineContext } from '@/engine'; -import { t, q, q0, m } from "@/lib/cli-rpc"; -import { editorSession, requireEditorSession, setEditorSession } from "./session"; -import { handleContextGet } from "./context"; -import { createAssetResolver, handleMediaProbe, handleMediaFrame, handleMediaTranscribe, handleMediaFilmstrip, handleMediaWaveform, handleMediaListen } from "./media"; -import { handleCapture } from "./capture"; -import { handleCheck } from "./check"; -import { handleExport } from "./export"; -import { handleLogs } from "./logs"; -import { handleModels } from "./models"; -import { handleVoices } from "./voices"; -import { cliBridge } from '@/lib/ipc'; -import { createRouterCaller } from '@/lib/cli-rpc'; -import { openProjectFolder } from '@/projects'; -import { projectRoute } from '@/hooks/use-project-route'; -import { assert } from "@/utils/common"; -import { handleWindowScreenshot } from "./window"; -import { useFullscreenState } from "@/hooks/use-fullscreen-state"; - -import type { JSX, Accessor } from 'solid-js'; -import type { Navigator } from '@solidjs/router'; -import type { User } from '@supabase/supabase-js'; - -type EditorApiProviderProps = { - children: JSX.Element; -}; - -type EditorApiContextValue = { - isFullscreen: Accessor; - isDesktop: boolean; -}; - -const EditorApiContext = createContext(); - -/** - * The one CLI router, registered for as long as the app runs. Every endpoint - * is reachable whether or not a project is open; the ones that need one read - * the session slot (see ./session) per request and fail with a clear error — - * or, for `context`, report that nothing is open. Renders nothing; must sit - * inside the router tree for `useNavigate` and inside the auth provider. - */ -export function EditorApi() { - const navigate = useNavigate(); - const auth = useAuth(); - - const requireAuth = (fn: (data: I) => Promise) => (data: I) => { - assert(auth.isAuthenticated(), "Sign in required: AI generation needs a Diffusion Studio account."); - return fn(data); - }; - - const getUser = () => { - const user = auth.user(); - assert(user, "User not found"); - return user; - }; - - const router = createAppRouter({ navigate, getUser, requireAuth }); - onCleanup(cliBridge.register(createRouterCaller(router))); - return null; -} - -/** - * Publishes the editor session for the CLI router while the project is open, - * and provides the editor UI's own view of the app shell (fullscreen state, - * desktop-ness). Mounted per project page. - */ -export function EditorApiProvider(props: EditorApiProviderProps) { - const project = useProject(); - const isFullscreen = useFullscreenState(); - const world = useWorld(); - const engine = useEngineContext(); - - createEffect(() => { - if (!window.desktop || project.id() !== world.get(Project)?.id) return; - - setEditorSession({ world, project, engine }); - onCleanup(() => setEditorSession(null)); - }); - - return ( - - {props.children} - - ); -} - - -type AppRouterDeps = { - navigate: Navigator; - getUser: () => User; - requireAuth: (fn: (data: I) => Promise) => (data: I) => Promise; -}; - -function createAppRouter({ navigate, getUser, requireAuth }: AppRouterDeps) { - const resolveAsset = createAssetResolver(editorSession); - - return t.router({ - ping: t.procedure.query(() => {}), - open: m(async ({ dir }: { dir: string }) => { - const project = await openProjectFolder(dir); - navigate(projectRoute(project.id || project.name)); - return { id: project.id, name: project.displayName, dir: project.dir }; - }), - whoami: t.procedure.query(() => getUser()), - context: q0(handleContextGet(editorSession)), - capture: q(handleCapture(requireEditorSession)), - check: q(handleCheck(requireEditorSession)), - export: m(handleExport(requireEditorSession)), - models: q(handleModels()), - logs: q(handleLogs()), - screenshot: q0(handleWindowScreenshot()), - voices: q0(handleVoices()), - media: t.router({ - probe: q(handleMediaProbe(resolveAsset)), - frame: q(handleMediaFrame(resolveAsset)), - transcribe: q(handleMediaTranscribe(resolveAsset)), - filmstrip: q(handleMediaFilmstrip(resolveAsset)), - waveform: q(handleMediaWaveform(resolveAsset)), - listen: q(requireAuth(handleMediaListen(resolveAsset, editorSession))), - }), - }); -} - -export type AppRouter = ReturnType; - -export function useEditorApi() { - const ctx = useContext(EditorApiContext); - assert(ctx, "useEditorApi must be used within EditorApiProvider"); - return ctx; -} diff --git a/apps/web/src/context/dapi/capture.ts b/apps/web/src/context/dapi/capture.ts deleted file mode 100644 index 375e7bae..00000000 --- a/apps/web/src/context/dapi/capture.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { - composeSheet, - createImageEncoder, - decodePngBase64, - planSheet, - planSheetSizes, - sheetTimecode, -} from "@diffusionstudio/encoder"; - -import { getParentNode, isScene, Source } from "@diffusionstudio/runtime"; - -import { createCapture } from "@/engine/capture"; -import { resolveNode } from "./nodes"; - -import type { CaptureRequest, CaptureResult, TimecodedImage } from "@diffusionstudio/dapi"; -import type { EditorSession } from "./session"; - -// Ceiling on the height a sheet cell renders a node at. -const SHEET_CAPTURE_HEIGHT = 1080; - -export function handleCapture(session: () => EditorSession) { - return async ({ id, frames, combine = true, perSheet }: CaptureRequest): Promise => { - const { world, project } = session(); - const node = resolveNode(world, id); - - // Scenes only: a capture's promise is that its frames are the frames an - // export encodes, and a scene is the unit an export renders — framing an - // arbitrary node would need its bounds measured across the requested - // positions first, and that pre-roll runs the project's code ahead of the - // frames being drawn, which is exactly what an export never does. - if (!isScene(node)) { - let scene = getParentNode(node); - while (scene !== null && !isScene(scene)) scene = getParentNode(scene); - const stamp = scene?.get(Source)?.value; - throw new Error( - stamp - ? `"${id}" is not a scene — capture renders what an export renders. Capture its scene "${stamp}" instead.` - : `"${id}" is not a scene — capture renders what an export renders, so it takes a scene id.`, - ); - } - - // `undefined` means the export's first frame (the workarea's start). - let shots = frames; - if (shots === undefined || shots.length === 0) { - shots = [0]; - } - - // The project re-rendered into a world of its own, reduced to this scene: - // the same arrangement an export runs against, and the encoder's to draw. - const capture = await createCapture(world, node, { dir: project.dir() }); - - try { - const encoder = await createImageEncoder(capture.world, { - frames: shots, - resolution: 720, - }); - - // Sheets render at their cell size instead of the flat 720p: with a few - // frames that is sharper than a standalone capture, never coarser. A - // scene is drawn, not decoded, so a small one is worth rendering past - // its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens. - const aspect = encoder.bounds.width / encoder.bounds.height; - const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT); - const sizes = combine ? planSheetSizes(shots.length, perSheet) : []; - const plans = sizes.map((n) => planSheet(n, { width: height * aspect, height })); - if (combine) { - encoder.resize(Math.max(...plans.map((plan) => plan.cellHeight))); - } - - const result = await encoder.render(); - - if (result.type === "canceled") throw new Error("Capture canceled"); - if (result.type === "error") throw result.error; - - if (!combine) return result.data; - - const sheets: TimecodedImage[] = []; - let offset = 0; - for (const [sheet, size] of sizes.entries()) { - const group = result.data.slice(offset, offset + size); - const cells = group.map(({ timecode }, k) => ({ at: shots[offset + k], timecode })); - offset += size; - const images = await Promise.all(group.map((image) => decodePngBase64(image.base64))); - sheets.push({ - timecode: sheetTimecode(cells), - base64: await composeSheet( - images.map((image, k) => ({ image, label: group[k].timecode })), - plans[sheet], - ), - }); - for (const image of images) image.close(); - } - return sheets; - } finally { - capture.dispose(); - } - }; -} diff --git a/apps/web/src/context/dapi/context.ts b/apps/web/src/context/dapi/context.ts deleted file mode 100644 index 1f6a7bb8..00000000 --- a/apps/web/src/context/dapi/context.ts +++ /dev/null @@ -1,99 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime"; - -import { getProjectsRoot } from "@/projects"; - -import type { Accessor } from "solid-js"; -import type { Entity, World } from "koota"; -import type { EditorSession } from "./session"; - -/** - * What `dapi context` reports: what the project's source cannot say. The JSX is - * the composition — its scenes, what is selected, which scene is active, the - * work area are all in the file, and a caller that wants them reads it. What is - * left over is which folder projects live under, which project folder the app - * has open, where its playhead sits, which font families are actually - * registered in the world drawing it. With no project open only the root is - * left to report, and the report says so. - */ -export function handleContextGet(session: Accessor) { - return async () => { - const rootDir = await getProjectsRoot(); - - const open = session(); - if (!open) return { rootDir, projectDir: null }; - - const { world, project } = open; - const frameRate = world.get(FrameRate)?.value || 30; - const active = getActiveEntity(world); - - return { - rootDir, - projectDir: project.dir(), - // Seconds, the unit the source places clips in; null when no scene is - // active, which is when there is no playhead to report. - currentTime: active ? (active.get(Computed)?.localTime ?? 0) / frameRate : null, - // What text can be drawn with right now: registered in the world, not - // merely named in the source. The editor default is always among them. - fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])], - generations: collectGenerations(world), - }; - } -} - -type GenerationRow = { - /** The element's source stamp, `:`; null for an - * entity no element produced. */ - element: string | null; - name: string | null; - state: "generating" | "failed" | "done"; - /** What the generation failed with, on `failed` rows. */ - error?: string; - /** The library path the generation landed as, on `done` rows — ready for - * `dapi media probe` and its siblings. */ - asset?: string; -}; - -function collectGenerations(world: World): GenerationRow[] { - const library = world.get(Library); - const rows: GenerationRow[] = []; - const seen = new Set(); - - const push = (row: GenerationRow) => { - const key = JSON.stringify(row); - if (seen.has(key)) return; - seen.add(key); - rows.push(row); - }; - - for (const entity of world.query(Generating)) { - push({ ...describeElement(entity), state: "generating" }); - } - - for (const entity of world.query(SourceError)) { - const failure = entity.get(SourceError)!; - if (!failure.generated) continue; - push({ ...describeElement(entity), state: "failed", error: failure.value }); - } - - for (const entity of world.query(AssetId)) { - // A stale binding — the element is off generating a new answer or failed - // getting one — is not a generation that is done. - if (entity.has(Generating) || entity.has(PendingSource) || entity.has(SourceError)) continue; - const asset = library?.get(entity.get(AssetId)!.value); - if (!asset?.generation) continue; - push({ ...describeElement(entity), state: "done", asset: asset.path }); - } - - return rows; -} - -function describeElement(entity: Entity): Pick { - return { - element: entity.get(Source)?.value || null, - name: entity.get(Name)?.value || null, - }; -} diff --git a/apps/web/src/context/dapi/export.ts b/apps/web/src/context/dapi/export.ts deleted file mode 100644 index c4e96e85..00000000 --- a/apps/web/src/context/dapi/export.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { canEncodeVideo } from "mediabunny"; -import { computeOutputSize } from "@diffusionstudio/encoder"; -import { Computed, FrameRate, getParentNode, isScene, Source, Workarea } from "@diffusionstudio/runtime"; - -import { renderOverlay, renderScene } from "@/context/render"; -import { ElectronWritableFileHandle } from "@/lib/electron-file-writable"; -import { ProjectConfig as ProjectConfigTrait } from "@/engine/traits"; -import { sceneConfigKey } from "@/engine/project-config"; -import { - getDefaultExportTemplate, - VIDEO_FORMAT_OPTIONS, -} from "@/components/sidebar-right/inspector/export-templates"; -import { mainBridge } from "@/lib/ipc"; -import { MAIN_CHANNELS } from "@desktop/main-channels"; -import { resolveNode } from "./nodes"; - -import type { ExportRequest, ExportResult, ExportSettings } from "@diffusionstudio/dapi"; -import type { ContainerFormat, ExportConfig } from "@/engine/project-config"; -import type { EditorSession } from "./session"; - -/** - * The container the export writes: the output extension when a path was - * given (ffmpeg's rule — the file must be what its name says), the config's - * format otherwise, mp4 when neither says. A given path settles it entirely, - * so a bad config format only fails an export that would actually use it. - */ -function resolveFormat(path: string | undefined, config: ExportConfig): ContainerFormat { - const supported = VIDEO_FORMAT_OPTIONS.map((format) => `.${format}`).join(", "); - if (path !== undefined) { - const extension = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); - if (!extension || !VIDEO_FORMAT_OPTIONS.includes(extension as ContainerFormat)) { - throw new Error(`The output path must end in a container extension: ${supported}.`); - } - return extension as ContainerFormat; - } - const format = config.format ?? "mp4"; - if (!VIDEO_FORMAT_OPTIONS.includes(format)) { - throw new Error(`The export entry names an unknown format "${format}" — use one of ${supported.replaceAll(".", "")}.`); - } - return format; -} - -export function handleExport(session: () => EditorSession) { - return async ({ id, path }: ExportRequest): Promise => { - const { world, project, engine } = session(); - - const scene = resolveNode(world, id); - if (!isScene(scene)) { - let parent = getParentNode(scene); - while (parent !== null && !isScene(parent)) parent = getParentNode(parent); - const stamp = parent?.get(Source)?.value; - throw new Error( - stamp - ? `"${id}" is not a scene — a scene is the unit an export renders. Export its scene "${stamp}" instead.` - : `"${id}" is not a scene — a scene is the unit an export renders, so export takes a scene id.`, - ); - } - - // The scene's entry in the project's package.json (`diffusion.export.`) - // — the same one the app's export panel writes — so a CLI export - // reproduces the in-app one; a scene without an entry uses the default - // template, the way ⌘E does. `template` is only the preset's label. - const base = world.get(ProjectConfigTrait)?.exportOf(scene) ?? getDefaultExportTemplate(); - const settings: ExportConfig = { format: base.format, video: base.video, audio: base.audio }; - - const format = resolveFormat(path, settings); - const key = sceneConfigKey(scene) ?? id; - const target = path ?? `${project.dir()}/exports/${key.replace(/[^\w.-]+/g, "-")}.${format}`; - - // Fail before the render machinery spins up: an unencodable configuration - // is known right away (same precheck the UI export runs). - const videoEnabled = format !== "ogg" && settings.video?.enabled !== false; - const computed = scene.get(Computed); - const { width, height } = computeOutputSize( - computed?.width || 1920, - computed?.height || 1080, - settings.video?.resolution ?? 1080, - ); - if (videoEnabled) { - const codec = settings.video?.codec ?? "avc"; - const encodable = await canEncodeVideo(codec, { - width, - height, - bitrate: settings.video?.bitrate ?? 10e6, - }); - if (!encodable) { - throw new Error( - `Cannot encode ${codec.toUpperCase()} at ${width}×${height}. ` + - "Set a lower resolution, a lower bitrate, or another codec in the scene's export entry.", - ); - } - } - - const workarea = scene.get(Workarea); - const frames = workarea ? workarea.end - workarea.start : computed?.duration ?? 0; - const duration = frames / (world.get(FrameRate)?.value || 30); - - // renderScene owns the one render slot (it stops the live engine and - // raises the progress overlay), so refuse a second export rather than - // interleave. No await sits between this check and renderScene claiming - // the overlay, so two racing requests cannot both pass. - if (renderOverlay()) { - throw new Error("An export is already running — wait for it to finish, or cancel it in the app."); - } - - const handle = new ElectronWritableFileHandle(target); - try { - const result = await renderScene(engine, { - scene, - target: handle, - config: { ...settings, format }, - dir: project.dir(), - }); - if (result.type === "canceled") throw new Error("Export canceled in the app"); - if (result.type === "error") throw result.error; - } catch (error) { - // Close the fd and drop the partial file; a failed export leaves nothing. - await handle.dispose().catch(() => {}); - throw error; - } - - const stat = await mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_STAT, { - dir: project.dir(), - source: target, - }); - - const config: ExportSettings = { format, video: settings.video, audio: settings.audio }; - return { - path: target, - width: videoEnabled ? width : 0, - height: videoEnabled ? height : 0, - duration, - size: stat?.size ?? 0, - config, - }; - }; -} diff --git a/apps/web/src/context/dapi/logs.ts b/apps/web/src/context/dapi/logs.ts deleted file mode 100644 index e45d2389..00000000 --- a/apps/web/src/context/dapi/logs.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { mainBridge } from "@/lib/ipc"; -import { MAIN_CHANNELS } from "@desktop/main-channels"; -import type { LogsRequest, LogLevel } from "@diffusionstudio/dapi"; - -const LEVEL_RANK: Record = { debug: 0, info: 1, warning: 2, error: 3 }; - -export function handleLogs() { - return async (req: LogsRequest) => { - let entries = await mainBridge.call(MAIN_CHANNELS.LOGS_GET, undefined); - if (req.level !== undefined) { - const min = LEVEL_RANK[req.level]; - entries = entries.filter((e) => LEVEL_RANK[e.level] >= min); - } - if (req.tail !== undefined) entries = entries.slice(-req.tail); - return entries; - }; -} diff --git a/apps/web/src/context/dapi/media.ts b/apps/web/src/context/dapi/media.ts deleted file mode 100644 index 09c643f9..00000000 --- a/apps/web/src/context/dapi/media.ts +++ /dev/null @@ -1,361 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny'; -import { pickInformativeTimes } from './frame-triage'; -import { trpc } from '@/lib/trpc'; -import { uploadBlob } from '@/lib/uploads'; -import { composeSheet, planSheet, planSheetSizes, sheetTimecode } from '@diffusionstudio/encoder'; -import { assert } from '@/utils'; -import { startResumableSession, uploadResumableStream } from '@/lib/uploads'; -import { filmstripAsset, formatTimecode, getAssetFile, getLibrary, Project, transcodeForAnalysis, transcodeForTranscription, waveformAsset } from '@diffusionstudio/runtime'; -import { AssetLibrary, assetName, isAbsoluteSource, isUrlSource } from '@diffusionstudio/assets'; -import { createProjectFS } from '@/projects/fs'; - -import type { Accessor } from 'solid-js'; -import type { Asset } from '@diffusionstudio/assets'; -import type { EditorSession } from './session'; -import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/dapi"; - -type ResolveAsset = (path: string) => Promise; - -/** - * Resolves a command target by path. With a project open, its library answers: - * library paths look assets up, absolute paths and URLs are described in place - * without being added (transient assets). With none open, a throwaway library - * over a project-less FS describes absolute paths and URLs the same way — a - * fresh one per request, so nothing is remembered between calls. - */ -export function createAssetResolver(session: Accessor): ResolveAsset { - return (path) => { - const world = session()?.world; - if (world) return getLibrary(world).resolve(path); - assert( - isAbsoluteSource(path) || isUrlSource(path), - `Could not resolve "${path}": with no project open only absolute paths and URLs resolve — run \`dapi open \` to use library paths.`, - ); - return new AssetLibrary(createProjectFS("")).resolve(path); - }; -} - -const PROBE_SAMPLE_PACKETS = 200; - -export function handleMediaProbe(resolve: ResolveAsset) { - return async (req: MediaProbeRequest): Promise => { - const asset = await resolve(req.path); - - const blob = await getAssetFile(asset); - const base = { - id: asset.id, - name: assetName(asset), - path: asset.path, - type: asset.type, - mimeType: asset.mimeType, - size: blob.size, - ...("width" in asset && { width: asset.width, height: asset.height }), - }; - - const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) }); - try { - const format = await input.getFormat(); - const mimeType = await input.getMimeType(); - const duration = await input.computeDuration(); - const { images, ...tags } = await input.getMetadataTags(); - delete tags.raw; - - const tracks: Array> = []; - for (const track of await input.getTracks()) { - const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS); - tracks.push({ - id: track.id, - type: track.type, - codec: track.codec, - language: track.languageCode, - firstTimestamp: await track.getFirstTimestamp(), - duration: await track.computeDuration(), - ...stats, - ...(track.isVideoTrack() && { - codedWidth: track.codedWidth, - codedHeight: track.codedHeight, - displayWidth: track.displayWidth, - displayHeight: track.displayHeight, - rotation: track.rotation, - }), - ...(track.isAudioTrack() && { - sampleRate: track.sampleRate, - channels: track.numberOfChannels, - }), - }); - } - - return { - ...base, - format: format.name, - mimeType, - duration, - tags: { ...tags, ...(images?.length && { attachedImages: images.length }) }, - tracks, - }; - } catch { - return { ...base, format: null, tracks: [] }; - } finally { - input.dispose(); - } - }; -} - -// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio -// preserved). A budget of 0 means native resolution. `small` keeps frames small -// enough for vision models and is the default. -const FRAME_QUALITY_BUDGETS = { - small: 384 * 384, // 147,456 - medium: 768 * 768, // 589,824 - large: 1536 * 1536, // 2,359,296 - fullres: 0, // native -} as const; - -// Default cap on frames returned by auto selection when `count` is not given. -const AUTO_MAX_FRAMES = 30; - -export function handleMediaFrame(resolve: ResolveAsset) { - return async (req: MediaFrameRequest): Promise => { - const { times, count, start, end, quality, auto } = req; - const combine = req.combine ?? true; - const asset = await resolve(req.path); - const id = asset.id; - assert(asset.type === "VIDEO", `Asset ${id} is not a video.`); - - // `count` samples evenly across a window (default the whole clip); `auto` - // scans the window and keeps frames where the footage settles into a new - // visual state, capped at `count` (resolved once the track is open). - // Otherwise grab the explicit `times` (falling back to a single frame at 0). - const from = Math.min(Math.max(start ?? 0, 0), asset.duration); - const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration); - let requested: number[] = []; - if (auto || count !== undefined) { - assert(to > from, `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`); - if (!auto && count !== undefined) { - const interval = (to - from) / count; - requested = Array.from({ length: count }, (_, i) => from + i * interval); - } - } else { - const raw = times && times.length ? times : [0]; - // A negative time is an offset back from the end of the clip: -1 is one - // second before the end, -1f one frame before it. - requested = raw.map((t) => { - if (t >= 0) { - assert(t <= asset.duration, `--time ${t}s is past the asset's duration (${asset.duration.toFixed(2)}s).`); - return t; - } - const resolved = asset.duration + t; - assert(resolved >= 0, `--time ${t} counts past the start of the clip (duration ${asset.duration.toFixed(2)}s).`); - return resolved; - }); - } - - const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")]; - - const blob = await getAssetFile(asset); - const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) }); - try { - const track = await input.getPrimaryVideoTrack(); - assert(track, `Asset ${id} has no video track.`); - - // Track timestamps may not start at 0; offset content time by the first. - const firstTimestamp = (await track.getFirstTimestamp()) ?? 0; - - if (auto) { - const picked = await pickInformativeTimes(track, { - from: firstTimestamp + from, - to: firstTimestamp + to, - max: count ?? AUTO_MAX_FRAMES, - }); - requested = picked.map((t) => Math.max(0, t - firstTimestamp)); - } - - // Downscale to fit the pixel budget while preserving aspect ratio; setting - // only the width lets the sink derive a matching height. - const displayWidth = await track.getDisplayWidth(); - const displayHeight = await track.getDisplayHeight(); - let sourceWidth = displayWidth; - let sourceHeight = displayHeight; - if (budget > 0 && displayWidth * displayHeight > budget) { - const scale = Math.sqrt(budget / (displayWidth * displayHeight)); - sourceWidth = Math.max(1, Math.round(displayWidth * scale)); - sourceHeight = Math.max(1, Math.round(displayHeight * scale)); - } - - // Lay the sheets out up front: the largest cell across them sets the - // decode size, so no frame is decoded bigger than it will be drawn. - const sizes = combine ? planSheetSizes(requested.length, req.perSheet) : []; - const plans = sizes.map((n) => planSheet(n, { width: sourceWidth, height: sourceHeight })); - const width = combine - ? Math.min(sourceWidth, Math.max(...plans.map((plan) => plan.cellWidth))) - : sourceWidth; - - // Decode in ascending order (the sink's fast path), remember each - // entry's original slot so output mirrors the requested order. - const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time); - - // No pool: each yielded canvas is fresh, so converting to PNG can't race - // the generator's read-ahead reusing a pooled canvas. - const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined); - const timestamps = ordered.map(({ time }) => firstTimestamp + time); - - // Which sheet each frame belongs to, and where that sheet starts. - const sheetOf: number[] = []; - const sheetStart: number[] = []; - for (const [sheet, size] of sizes.entries()) { - sheetStart.push(sheetOf.length); - for (let k = 0; k < size; k++) sheetOf.push(sheet); - } - const missing = [...sizes]; - - const cells: Array<{ at: number; timecode: string }> = new Array(requested.length); - const canvases: Array = new Array(requested.length); - const result: TimecodedImage[] = new Array(combine ? sizes.length : requested.length); - - let i = 0; - for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) { - const { time, index } = ordered[i++]; - assert(wrapped, `No frame found at ${time}s.`); - const timecode = formatTimecode(time, asset.frameRate); - cells[index] = { at: time, timecode }; - - if (!combine) { - result[index] = { timecode, base64: await canvasToPngBase64(wrapped.canvas) }; - continue; - } - - // Compose a sheet as soon as its last frame lands and drop the - // canvases, so a long run never holds every decoded frame at once. - canvases[index] = wrapped.canvas; - const sheet = sheetOf[index]; - if (--missing[sheet] > 0) continue; - - const from = sheetStart[sheet]; - const to = from + sizes[sheet]; - result[sheet] = { - timecode: sheetTimecode(cells.slice(from, to)), - base64: await composeSheet( - canvases.slice(from, to).map((canvas, k) => ({ image: canvas!, label: cells[from + k].timecode })), - plans[sheet], - ), - }; - for (let k = from; k < to; k++) canvases[k] = undefined; - } - - return result; - } finally { - input.dispose(); - } - }; -} - -const transcripts = new Map(); - -export function handleMediaTranscribe(resolve: ResolveAsset) { - return async (req: MediaTranscribeRequest): Promise => { - const asset = await resolve(req.path); - const id = asset.id; - assert( - asset.type === "AUDIO" || asset.type === "VIDEO", - `Asset ${id} is not a video or audio asset.`, - ); - - let transcript = transcripts.get(asset.id); - if (!transcript) { - const uploadId = crypto.randomUUID(); - const audioFile = await transcodeForTranscription(asset); - const fileRef = await uploadBlob(audioFile, uploadId); - if (!fileRef) throw new Error(`Failed to upload asset ${id} for transcription.`); - - ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef })); - if (!transcript.length || transcript.every((s) => s.words.length === 0)) { - throw new Error("No speech detected. The audio does not appear to contain recognizable speech."); - } - - transcripts.set(asset.id, transcript); - } - - return { segments: transcript }; - }; -} - -export function handleMediaFilmstrip(resolve: ResolveAsset) { - return async (req: MediaFilmstripRequest): Promise => { - const asset = await resolve(req.path); - const { dataUrl, ...rest } = await filmstripAsset(asset, { start: req.start, end: req.end, scale: req.scale }); - const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1); - return { base64, ...rest }; - }; -} - -export function handleMediaWaveform(resolve: ResolveAsset) { - return async (req: MediaWaveformRequest): Promise => { - const asset = await resolve(req.path); - const { dataUrl, ...rest } = await waveformAsset(asset, { start: req.start, end: req.end, scale: req.scale }); - const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1); - return { base64, ...rest }; - }; -} - -export function handleMediaListen(resolve: ResolveAsset, session: Accessor) { - return async (req: MediaListenRequest): Promise => { - const { prompt, start, end } = req; - let { stripVideo } = req; - const asset = await resolve(req.path); - const id = asset.id; - assert( - asset.type === "AUDIO" || asset.type === "VIDEO", - `Asset ${id} is not a video or audio asset.`, - ); - - const hasWindow = start !== undefined || end !== undefined; - stripVideo = stripVideo !== false && asset.type === "VIDEO"; - - const contentType = - asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg"; - - const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : ""; - const uploadId = `${session()?.world.get(Project)?.id ?? "project"}-${id}-analyze${stripVideo ? "-audio" : ""}${window}` - .replace(/[^A-Za-z0-9._-]/g, "_"); - const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({ - action: "resumable", - id: uploadId, - contentType, - }); - - // Transcoding pending - if (uploadUrl) { - const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo }); - const sessionUrl = await startResumableSession(uploadUrl, contentType); - const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl); - await transcoder.run?.(); - await uploadPromise; - } - - const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt }); - - return { result: analysis, start, end }; - }; -} - -async function canvasToPngBase64(canvas: HTMLCanvasElement | OffscreenCanvas): Promise { - if (canvas instanceof OffscreenCanvas) { - const blob = await canvas.convertToBlob({ type: "image/png" }); - return base64FromArrayBuffer(await blob.arrayBuffer()); - } - return canvas.toDataURL("image/png").split(",")[1] ?? ""; -} - -function base64FromArrayBuffer(buf: ArrayBuffer): string { - const bytes = new Uint8Array(buf); - let binary = ""; - const chunk = 0x8000; - for (let i = 0; i < bytes.length; i += chunk) { - binary += String.fromCharCode(...bytes.subarray(i, i + chunk)); - } - return btoa(binary); -} diff --git a/apps/web/src/context/dapi/models.ts b/apps/web/src/context/dapi/models.ts deleted file mode 100644 index 2b5b07f5..00000000 --- a/apps/web/src/context/dapi/models.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { - PROMPT_INPUT_IMAGE_MODEL_OPTIONS, - PROMPT_INPUT_VIDEO_MODEL_OPTIONS, - PROMPT_INPUT_AUDIO_MODEL_OPTIONS, -} from "@/components/genai/config"; -import type { ModelsRequest, ModelInfo } from "@diffusionstudio/dapi"; - -export function handleModels() { - return async (req: ModelsRequest) => { - const out: ModelInfo[] = []; - if (!req.type || req.type === "image") { - for (const option of PROMPT_INPUT_IMAGE_MODEL_OPTIONS) { - out.push({ - type: "image", - id: option.id, - name: option.name, - }); - } - } - if (!req.type || req.type === "video") { - for (const option of PROMPT_INPUT_VIDEO_MODEL_OPTIONS) { - out.push({ - type: "video", - id: option.id, - name: option.name, - durations: option.durations, - aspectRatios: option.aspectRatios, - features: option.features, - }); - } - } - if (!req.type || req.type === "audio") { - for (const option of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) { - out.push({ - type: "audio", - id: option.id, - name: option.name, - }); - } - } - return out; - }; -} diff --git a/apps/web/src/context/dapi/window.ts b/apps/web/src/context/dapi/window.ts deleted file mode 100644 index 5ae14da0..00000000 --- a/apps/web/src/context/dapi/window.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { mainBridge } from "@/lib/ipc"; -import { MAIN_CHANNELS } from "@desktop/main-channels"; - -export function handleGetFullscreenState() { - if (!window.desktop) return Promise.resolve(false); - return mainBridge.call(MAIN_CHANNELS.WINDOW_IS_FULLSCREEN, undefined); -} - -export function handleWindowScreenshot() { - return () => mainBridge.call(MAIN_CHANNELS.WINDOW_CAPTURE, undefined); -} - -export function handleWindowFullscreenChange(mutate: (fullscreen: boolean) => void) { - return ({ fullscreen }: { fullscreen: boolean }) => mutate(fullscreen); -} diff --git a/apps/web/src/dapi/api.tsx b/apps/web/src/dapi/api.tsx new file mode 100644 index 00000000..611f2246 --- /dev/null +++ b/apps/web/src/dapi/api.tsx @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { createEffect, createContext, useContext, onCleanup } from "solid-js"; +import { useNavigate } from "@solidjs/router"; +import { useWorld } from "@diffusionstudio/koota-solid"; +import { Project } from "@diffusionstudio/runtime"; +import { DapiError } from "@diffusionstudio/dapi"; +import { useProject } from "@/context/project"; +import { useAuth } from "@/context/auth"; +import { useEngineContext } from "@/engine"; +import { openProjectFolder } from "@/projects"; +import { projectRoute } from "@/hooks/use-project-route"; +import { useFullscreenState } from "@/hooks/use-fullscreen-state"; +import { assert } from "@/utils/common"; +import { cliBridge } from "./bridge"; +import { handlers } from "./handlers"; +import { editorSession, requireEditorSession, setEditorSession } from "./session"; + +import type { JSX, Accessor } from "solid-js"; +import type { ToolContext } from "./handler"; + +type EditorApiContextValue = { + isFullscreen: Accessor; + isDesktop: boolean; +}; + +const EditorApiContext = createContext(); + +/** + * Registers the tool handlers for as long as the app runs. Every tool is + * reachable whether or not a project is open; the ones that need one read + * the session slot (see ./session) per call and fail with a clear error — + * or, for `context`, report that nothing is open. Renders nothing; must sit + * inside the router tree for `useNavigate` and inside the auth provider. + */ +export function EditorApi() { + const navigate = useNavigate(); + const auth = useAuth(); + + const context = (signal: AbortSignal): ToolContext => ({ + session: editorSession, + requireSession: requireEditorSession, + signal, + app: { + async openProject(dir) { + const project = await openProjectFolder(dir); + navigate(projectRoute(project.id || project.name)); + return { id: project.id, name: project.displayName, dir: project.dir }; + }, + user: () => auth.user() ?? null, + requireUser() { + const user = auth.user(); + if (!user) throw new DapiError("sign-in-required", "Sign in required: AI generation needs a Diffusion Studio account."); + return user; + }, + }, + }); + + onCleanup(cliBridge.register(handlers, context)); + return null; +} + +/** + * Publishes the editor session for the tool handlers while the project is + * open, and provides the editor UI's own view of the app shell (fullscreen + * state, desktop-ness). Mounted per project page. + */ +export function EditorApiProvider(props: { children: JSX.Element }) { + const project = useProject(); + const isFullscreen = useFullscreenState(); + const world = useWorld(); + const engine = useEngineContext(); + + createEffect(() => { + if (!window.desktop || project.id() !== world.get(Project)?.id) return; + + setEditorSession({ world, project, engine }); + onCleanup(() => setEditorSession(null)); + }); + + return ( + + {props.children} + + ); +} + +export function useEditorApi() { + const ctx = useContext(EditorApiContext); + assert(ctx, "useEditorApi must be used within EditorApiProvider"); + return ctx; +} diff --git a/apps/web/src/dapi/bridge.ts b/apps/web/src/dapi/bridge.ts new file mode 100644 index 00000000..2db5a591 --- /dev/null +++ b/apps/web/src/dapi/bridge.ts @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { parseToolArgs } from "@diffusionstudio/dapi"; +import { CLI_WIRE, encodeReply } from "@diffusionstudio/cli/protocol"; + +import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol"; +import type { Handlers, ServedToolName, ToolContext } from "./handler"; + +type Pending = { req: CliRequest; ws: WebSocket }; + +/** Builds the per-call context; the bridge supplies the abort signal. */ +export type ContextFactory = (signal: AbortSignal) => ToolContext; + +/** + * Answers tool calls from the CLI. Each CLI command hosts a short-lived + * WebSocket server; main relays only the connect info (CLI_WIRE.CONNECT) and + * we dial the CLI directly, so payloads never pass through main. One request + * and one reply per connection. + * + * The bridge is transport: it validates arguments against the catalog, runs + * the handler, and encodes the reply. Requests arriving before the handlers + * register (page bootstrap) are held and answered on registration. + */ +class CliBridge { + private handlers: Handlers | null = null; + private context: ContextFactory | null = null; + private held: Pending[] = []; + + constructor() { + // Bind eagerly so CONNECT arrivals during page bootstrap are caught + // rather than silently dropped before the handlers register. + window.desktop?.on(CLI_WIRE.CONNECT, (payload) => { + const { port, token } = payload as CliHandshake; + const ws = new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`); + ws.onmessage = (event) => { + try { + const req = JSON.parse(event.data as string) as CliRequest; + void this.dispatch({ req, ws }); + } catch (err) { + console.error("[cli-bridge] malformed CLI request", err); + ws.close(); + } + }; + }); + } + + register(handlers: Handlers, context: ContextFactory): () => void { + this.handlers = handlers; + this.context = context; + const held = this.held; + this.held = []; + for (const pending of held) void this.dispatch(pending); + return () => { + if (this.handlers === handlers) { + this.handlers = null; + this.context = null; + } + }; + } + + private async dispatch(pending: Pending): Promise { + const { req, ws } = pending; + // Liveness, answered by the transport itself: `open` waits on it after + // launching the app, before any handler exists to answer anything else. + if (req.path === "ping") { + ws.send(encodeReply({ ok: true, data: undefined })); + return; + } + if (!this.handlers || !this.context) { + this.held.push(pending); + return; + } + + const controller = new AbortController(); + ws.addEventListener("close", () => controller.abort(), { once: true }); + + let reply: CliReply; + try { + const name = req.path as ServedToolName; + const handler = this.handlers[name]; + if (!handler) throw new Error(`Unknown tool "${req.path}"`); + const args = parseToolArgs(name, req.input); + // Every handler takes its own parsed args; the map's union type cannot + // express that pairing, so the call site widens. + const data = await (handler as (args: unknown, ctx: ToolContext) => Promise)(args, this.context(controller.signal)); + reply = { ok: true, data }; + } catch (err) { + reply = { ok: false, error: (err as Error).message }; + } + try { + ws.send(encodeReply(reply)); + } catch (err) { + ws.send(encodeReply({ ok: false, error: `Failed to serialize reply for ${req.path}: ${(err as Error).message}` })); + } + } +} + +export const cliBridge = new CliBridge(); diff --git a/apps/web/src/dapi/handler.ts b/apps/web/src/dapi/handler.ts new file mode 100644 index 00000000..1a7f67ab --- /dev/null +++ b/apps/web/src/dapi/handler.ts @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { Accessor } from "solid-js"; +import type { User } from "@supabase/supabase-js"; +import type { ToolArgs, ToolName, ToolResult } from "@diffusionstudio/dapi"; +import type { EditorSession } from "./session"; + +/** + * What every handler gets besides its arguments. Built per call by the + * bridge from what the app shell provides (see ./api), so handlers are plain + * functions with no Solid in them. + */ +export type ToolContext = { + /** The open project's session, or null at the dashboard. */ + session: Accessor; + /** The session, or a `no-project` error the caller can act on. */ + requireSession(): EditorSession; + /** Fires when the caller goes away before the reply. */ + signal: AbortSignal; + /** What only the app shell can do: navigate, and know who is signed in. */ + app: { + openProject(dir: string): Promise>; + user(): User | null; + /** The signed-in user, or a `sign-in-required` error. */ + requireUser(): User; + }; +}; + +export type ToolHandler = (args: ToolArgs, ctx: ToolContext) => Promise>; + +/** + * The tools the renderer answers. `logs` is a main-process tool in the + * catalog and is forwarded from here until main hosts the server itself; + * `fonts`, `fetch` and `report` have no renderer side at all. + */ +export type ServedToolName = Exclude; + +export type Handlers = { readonly [N in ServedToolName]: ToolHandler }; diff --git a/apps/web/src/dapi/handlers/capture.ts b/apps/web/src/dapi/handlers/capture.ts new file mode 100644 index 00000000..13b1ee41 --- /dev/null +++ b/apps/web/src/dapi/handlers/capture.ts @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { createImageEncoder, decodePng } from "@diffusionstudio/encoder"; +import { DapiError } from "@diffusionstudio/dapi"; +import { createCapture } from "@/engine/capture"; +import { requireScene } from "../lib/scene"; +import { SheetCollector } from "../lib/sheets"; + +import type { ToolHandler } from "../handler"; + +// Ceiling on the height a sheet cell renders a node at. +const SHEET_CAPTURE_HEIGHT = 1080; + +export const capture: ToolHandler<"capture"> = async ({ id, frames, combine, perSheet }, ctx) => { + const { world, project } = ctx.requireSession(); + const scene = requireScene(world, id, "capture"); + + // `undefined` means the export's first frame (the workarea's start). + const shots = frames && frames.length > 0 ? frames : [0]; + + // The project re-rendered into a world of its own, reduced to this scene: + // the same arrangement an export runs against, and the encoder's to draw. + const target = await createCapture(world, scene, { dir: project.dir() }); + try { + const encoder = await createImageEncoder(target.world, { frames: shots, resolution: 720 }); + + // Sheets render at their cell size instead of the flat 720p: with a few + // frames that is sharper than a standalone capture, never coarser. A + // scene is drawn, not decoded, so a small one is worth rendering past + // its own size; beyond SHEET_CAPTURE_HEIGHT that only costs tokens. + let sheets: SheetCollector | undefined; + if (combine) { + const aspect = encoder.bounds.width / encoder.bounds.height; + const height = Math.max(encoder.bounds.height, SHEET_CAPTURE_HEIGHT); + sheets = new SheetCollector(shots.length, { width: height * aspect, height }, perSheet); + encoder.resize(sheets.cellHeight); + } + + const result = await encoder.render(); + if (result.type === "canceled") throw new DapiError("canceled", "Capture canceled"); + if (result.type === "error") throw result.error; + if (!sheets) return result.data; + + for (const [index, { timecode, png }] of result.data.entries()) { + await sheets.add(index, { at: shots[index]!, timecode, image: await decodePng(png) }); + } + return sheets.result(); + } finally { + target.dispose(); + } +}; diff --git a/apps/web/src/context/dapi/check.ts b/apps/web/src/dapi/handlers/check.ts similarity index 95% rename from apps/web/src/context/dapi/check.ts rename to apps/web/src/dapi/handlers/check.ts index 0aabb6df..03209cb7 100644 --- a/apps/web/src/context/dapi/check.ts +++ b/apps/web/src/dapi/handlers/check.ts @@ -8,11 +8,11 @@ import { SourceError, Workarea, framesToSeconds, getIntrinsicPaint, isText, } from "@diffusionstudio/runtime"; -import { resolveNode } from "./nodes"; +import { resolveNode } from "../lib/nodes"; -import type { CheckIssue, CheckRequest, CheckResult } from "@diffusionstudio/dapi"; +import type { CheckIssue } from "@diffusionstudio/dapi"; import type { Entity } from "koota"; -import type { EditorSession } from "./session"; +import type { ToolHandler } from "../handler"; // Absolute frames, [start, end). type Interval = { start: number; end: number }; @@ -147,9 +147,8 @@ function findGaps(window: Interval, coverage: Interval[]): Interval[] { return gaps.filter((gap) => gap.end - gap.start >= 1); } -export function handleCheck(session: () => EditorSession) { - return async ({ id }: CheckRequest): Promise => { - const { world } = session(); +export const check: ToolHandler<"check"> = async ({ id }, ctx) => { + const { world } = ctx.requireSession(); const target = resolveNode(world, id); const fps = world.get(FrameRate)?.value ?? 30; @@ -207,5 +206,4 @@ export function handleCheck(session: () => EditorSession) { }, issues, }; - }; -} +}; diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.ts new file mode 100644 index 00000000..d75b349c --- /dev/null +++ b/apps/web/src/dapi/handlers/context.ts @@ -0,0 +1,82 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime"; +import { getProjectsRoot } from "@/projects"; + +import type { Entity, World } from "koota"; +import type { GenerationRow } from "@diffusionstudio/dapi"; +import type { ToolHandler } from "../handler"; + +/** + * What the project's source cannot say. The JSX is the composition — its + * scenes, what is selected, which scene is active, the work area are all in + * the file, and a caller that wants them reads it. What is left over is which + * folder projects live under, which project folder the app has open, where its + * playhead sits, which font families are actually registered in the world + * drawing it. With no project open only the root is left to report. + */ +export const context: ToolHandler<"context"> = async (_, ctx) => { + const rootDir = await getProjectsRoot(); + + const open = ctx.session(); + if (!open) return { rootDir, projectDir: null }; + + const { world, project } = open; + const frameRate = world.get(FrameRate)?.value || 30; + const active = getActiveEntity(world); + + return { + rootDir, + projectDir: project.dir(), + // Seconds, the unit the source places clips in; null when no scene is + // active, which is when there is no playhead to report. + currentTime: active ? (active.get(Computed)?.localTime ?? 0) / frameRate : null, + // What text can be drawn with right now: registered in the world, not + // merely named in the source. The editor default is always among them. + fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])], + generations: collectGenerations(world), + }; +}; + +function collectGenerations(world: World): GenerationRow[] { + const library = world.get(Library); + const rows: GenerationRow[] = []; + const seen = new Set(); + + const push = (row: GenerationRow) => { + const key = JSON.stringify(row); + if (seen.has(key)) return; + seen.add(key); + rows.push(row); + }; + + for (const entity of world.query(Generating)) { + push({ ...describeElement(entity), state: "generating" }); + } + + for (const entity of world.query(SourceError)) { + const failure = entity.get(SourceError)!; + if (!failure.generated) continue; + push({ ...describeElement(entity), state: "failed", error: failure.value }); + } + + for (const entity of world.query(AssetId)) { + // A stale binding — the element is off generating a new answer or failed + // getting one — is not a generation that is done. + if (entity.has(Generating) || entity.has(PendingSource) || entity.has(SourceError)) continue; + const asset = library?.get(entity.get(AssetId)!.value); + if (!asset?.generation) continue; + push({ ...describeElement(entity), state: "done", asset: asset.path }); + } + + return rows; +} + +function describeElement(entity: Entity): Pick { + return { + element: entity.get(Source)?.value || null, + name: entity.get(Name)?.value || null, + }; +} diff --git a/apps/web/src/dapi/handlers/export.ts b/apps/web/src/dapi/handlers/export.ts new file mode 100644 index 00000000..fc688cc8 --- /dev/null +++ b/apps/web/src/dapi/handlers/export.ts @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { canEncodeVideo } from "mediabunny"; +import { computeOutputSize } from "@diffusionstudio/encoder"; +import { Computed, FrameRate, Workarea } from "@diffusionstudio/runtime"; +import { DapiError } from "@diffusionstudio/dapi"; + +import { renderOverlay, renderScene } from "@/context/render"; +import { ElectronWritableFileHandle } from "@/lib/electron-file-writable"; +import { ProjectConfig as ProjectConfigTrait } from "@/engine/traits"; +import { sceneConfigKey } from "@/engine/project-config"; +import { getDefaultExportTemplate, VIDEO_FORMAT_OPTIONS } from "@/components/sidebar-right/inspector/export-templates"; +import { mainBridge } from "@/lib/ipc"; +import { MAIN_CHANNELS } from "@desktop/main-channels"; +import { requireScene } from "../lib/scene"; + +import type { ExportSettings } from "@diffusionstudio/dapi"; +import type { ContainerFormat, ExportConfig } from "@/engine/project-config"; +import type { ToolHandler } from "../handler"; + +/** + * The container the export writes: the output extension when a path was + * given (ffmpeg's rule — the file must be what its name says), the config's + * format otherwise, mp4 when neither says. A given path settles it entirely, + * so a bad config format only fails an export that would actually use it. + */ +function resolveFormat(path: string | undefined, config: ExportConfig): ContainerFormat { + const supported = VIDEO_FORMAT_OPTIONS.map((format) => `.${format}`).join(", "); + if (path !== undefined) { + const extension = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); + if (!extension || !VIDEO_FORMAT_OPTIONS.includes(extension as ContainerFormat)) { + throw new DapiError("invalid-input", `The output path must end in a container extension: ${supported}.`); + } + return extension as ContainerFormat; + } + const format = config.format ?? "mp4"; + if (!VIDEO_FORMAT_OPTIONS.includes(format)) { + throw new DapiError( + "invalid-input", + `The export entry names an unknown format "${format}" — use one of ${supported.replaceAll(".", "")}.`, + ); + } + return format; +} + +export const exportScene: ToolHandler<"export"> = async ({ id, path }, ctx) => { + const { world, project, engine } = ctx.requireSession(); + const scene = requireScene(world, id, "export"); + + // The scene's entry in the project's package.json (`diffusion.export.`) + // — the same one the app's export panel writes — so a tool export + // reproduces the in-app one; a scene without an entry uses the default + // template, the way ⌘E does. `template` is only the preset's label. + const base = world.get(ProjectConfigTrait)?.exportOf(scene) ?? getDefaultExportTemplate(); + const settings: ExportConfig = { format: base.format, video: base.video, audio: base.audio }; + + const format = resolveFormat(path, settings); + const key = sceneConfigKey(scene) ?? id; + const target = path ?? `${project.dir()}/exports/${key.replace(/[^\w.-]+/g, "-")}.${format}`; + + // Fail before the render machinery spins up: an unencodable configuration + // is known right away (same precheck the UI export runs). + const videoEnabled = format !== "ogg" && settings.video?.enabled !== false; + const computed = scene.get(Computed); + const { width, height } = computeOutputSize( + computed?.width || 1920, + computed?.height || 1080, + settings.video?.resolution ?? 1080, + ); + if (videoEnabled) { + const codec = settings.video?.codec ?? "avc"; + const encodable = await canEncodeVideo(codec, { width, height, bitrate: settings.video?.bitrate ?? 10e6 }); + if (!encodable) { + throw new DapiError( + "unsupported", + `Cannot encode ${codec.toUpperCase()} at ${width}×${height}. ` + + "Set a lower resolution, a lower bitrate, or another codec in the scene's export entry.", + ); + } + } + + const workarea = scene.get(Workarea); + const frames = workarea ? workarea.end - workarea.start : computed?.duration ?? 0; + const duration = frames / (world.get(FrameRate)?.value || 30); + + // renderScene owns the one render slot (it stops the live engine and + // raises the progress overlay), so refuse a second export rather than + // interleave. No await sits between this check and renderScene claiming + // the overlay, so two racing requests cannot both pass. + if (renderOverlay()) { + throw new DapiError("busy", "An export is already running — wait for it to finish, or cancel it in the app."); + } + + const handle = new ElectronWritableFileHandle(target); + try { + const result = await renderScene(engine, { scene, target: handle, config: { ...settings, format }, dir: project.dir() }); + if (result.type === "canceled") throw new DapiError("canceled", "Export canceled in the app"); + if (result.type === "error") throw result.error; + } catch (error) { + // Close the fd and drop the partial file; a failed export leaves nothing. + await handle.dispose().catch(() => {}); + throw error; + } + + const stat = await mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_STAT, { dir: project.dir(), source: target }); + + const config: ExportSettings = { format, video: settings.video, audio: settings.audio }; + return { + path: target, + width: videoEnabled ? width : 0, + height: videoEnabled ? height : 0, + duration, + size: stat?.size ?? 0, + config, + }; +}; diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.ts new file mode 100644 index 00000000..2277306b --- /dev/null +++ b/apps/web/src/dapi/handlers/index.ts @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { open } from "./open"; +import { context } from "./context"; +import { capture } from "./capture"; +import { check } from "./check"; +import { exportScene } from "./export"; +import { models } from "./models"; +import { voices } from "./voices"; +import { whoami } from "./whoami"; +import { logs } from "./logs"; +import { screenshot } from "./screenshot"; +import { mediaProbe } from "./media-probe"; +import { mediaGrab } from "./media-grab"; +import { mediaTranscribe } from "./media-transcribe"; +import { mediaFilmstrip } from "./media-filmstrip"; +import { mediaWaveform } from "./media-waveform"; +import { mediaListen } from "./media-listen"; + +import type { Handlers } from "../handler"; + +/** Every tool the renderer answers, keyed by its catalog name. */ +export const handlers: Handlers = { + open, + context, + capture, + check, + export: exportScene, + models, + voices, + whoami, + logs, + screenshot, + media_probe: mediaProbe, + media_grab: mediaGrab, + media_transcribe: mediaTranscribe, + media_filmstrip: mediaFilmstrip, + media_waveform: mediaWaveform, + media_listen: mediaListen, +}; diff --git a/apps/web/src/dapi/handlers/logs.ts b/apps/web/src/dapi/handlers/logs.ts new file mode 100644 index 00000000..cdb909f8 --- /dev/null +++ b/apps/web/src/dapi/handlers/logs.ts @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mainBridge } from "@/lib/ipc"; +import { MAIN_CHANNELS } from "@desktop/main-channels"; + +import type { LogLevel } from "@diffusionstudio/dapi"; +import type { ToolHandler } from "../handler"; + +const LEVEL_RANK: Record = { debug: 0, info: 1, warning: 2, error: 3 }; + +// Main owns the buffer; this is a forward until main answers tools itself. +export const logs: ToolHandler<"logs"> = async ({ tail, level }) => { + let entries = await mainBridge.call(MAIN_CHANNELS.LOGS_GET, undefined); + if (level !== undefined) { + const min = LEVEL_RANK[level]; + entries = entries.filter((e) => LEVEL_RANK[e.level] >= min); + } + if (tail !== undefined) entries = entries.slice(-tail); + return entries; +}; diff --git a/apps/web/src/dapi/handlers/media-filmstrip.ts b/apps/web/src/dapi/handlers/media-filmstrip.ts new file mode 100644 index 00000000..7902b328 --- /dev/null +++ b/apps/web/src/dapi/handlers/media-filmstrip.ts @@ -0,0 +1,15 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { filmstripAsset } from "@diffusionstudio/runtime"; +import { resolveAsset } from "../lib/assets"; +import { dataUrlToBytes } from "../lib/png"; + +import type { ToolHandler } from "../handler"; + +export const mediaFilmstrip: ToolHandler<"media_filmstrip"> = async ({ path, start, end, scale }, ctx) => { + const asset = await resolveAsset(ctx, path); + const { dataUrl, ...rest } = await filmstripAsset(asset, { start, end, scale }); + return { png: dataUrlToBytes(dataUrl), ...rest }; +}; diff --git a/apps/web/src/dapi/handlers/media-grab.ts b/apps/web/src/dapi/handlers/media-grab.ts new file mode 100644 index 00000000..ed12e9fb --- /dev/null +++ b/apps/web/src/dapi/handlers/media-grab.ts @@ -0,0 +1,132 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { ALL_FORMATS, BlobSource, CanvasSink, Input } from "mediabunny"; +import { encodePng } from "@diffusionstudio/encoder"; +import { formatTimecode, getAssetFile } from "@diffusionstudio/runtime"; +import { DapiError } from "@diffusionstudio/dapi"; +import { pickInformativeTimes } from "../lib/frame-triage"; +import { requireAssetType, resolveAsset } from "../lib/assets"; +import { SheetCollector } from "../lib/sheets"; + +import type { FrameQuality, TimecodedImage } from "@diffusionstudio/dapi"; +import type { ToolHandler } from "../handler"; + +// Named quality presets mapped to a per-frame total-pixel budget (aspect ratio +// preserved). A budget of 0 means native resolution. `small` keeps frames small +// enough for vision models. +const FRAME_QUALITY_BUDGETS: Record = { + small: 384 * 384, + medium: 768 * 768, + large: 1536 * 1536, + fullres: 0, +}; + +// Default cap on frames returned by auto selection when `count` is not given. +const AUTO_MAX_FRAMES = 30; + +export const mediaGrab: ToolHandler<"media_grab"> = async (args, ctx) => { + const { times, count, start, end, quality, auto, combine, perSheet } = args; + const asset = await resolveAsset(ctx, args.path); + requireAssetType(asset, ["VIDEO"], "a video"); + + // `count` samples evenly across a window (default the whole clip); `auto` + // scans the window and keeps frames where the footage settles into a new + // visual state, capped at `count` (resolved once the track is open). + // Otherwise grab the explicit `times` (falling back to a single frame at 0). + const from = Math.min(Math.max(start ?? 0, 0), asset.duration); + const to = Math.min(Math.max(end ?? asset.duration, from), asset.duration); + let requested: number[] = []; + if (auto || count !== undefined) { + if (to <= from) { + throw new DapiError( + "invalid-input", + `The requested window is empty; start (${from.toFixed(2)}s) is at or past end (${to.toFixed(2)}s).`, + ); + } + if (!auto && count !== undefined) { + const interval = (to - from) / count; + requested = Array.from({ length: count }, (_, i) => from + i * interval); + } + } else { + requested = (times && times.length ? times : [0]).map((t) => resolveTime(t, asset.duration)); + } + + const budget = FRAME_QUALITY_BUDGETS[quality ?? (combine ? "fullres" : "small")]; + + const blob = await getAssetFile(asset); + const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) }); + try { + const track = await input.getPrimaryVideoTrack(); + if (!track) throw new DapiError("wrong-kind", `Asset ${asset.id} has no video track.`); + + // Track timestamps may not start at 0; offset content time by the first. + const firstTimestamp = (await track.getFirstTimestamp()) ?? 0; + + if (auto) { + const picked = await pickInformativeTimes(track, { + from: firstTimestamp + from, + to: firstTimestamp + to, + max: count ?? AUTO_MAX_FRAMES, + }); + requested = picked.map((t) => Math.max(0, t - firstTimestamp)); + } + + // Downscale to fit the pixel budget while preserving aspect ratio. + const displayWidth = await track.getDisplayWidth(); + const displayHeight = await track.getDisplayHeight(); + let sourceWidth = displayWidth; + let sourceHeight = displayHeight; + if (budget > 0 && displayWidth * displayHeight > budget) { + const scale = Math.sqrt(budget / (displayWidth * displayHeight)); + sourceWidth = Math.max(1, Math.round(displayWidth * scale)); + sourceHeight = Math.max(1, Math.round(displayHeight * scale)); + } + + // Lay the sheets out up front: the largest cell across them sets the + // decode size, so no frame is decoded bigger than it will be drawn. + const sheets = combine ? new SheetCollector(requested.length, { width: sourceWidth, height: sourceHeight }, perSheet) : undefined; + const width = sheets ? Math.min(sourceWidth, sheets.cellWidth) : sourceWidth; + + // Decode in ascending order (the sink's fast path), remember each + // entry's original slot so output mirrors the requested order. + const ordered = requested.map((time, index) => ({ time, index })).sort((a, b) => a.time - b.time); + + // No pool: each yielded canvas is fresh, so encoding it can't race the + // generator's read-ahead reusing a pooled canvas. + const sink = new CanvasSink(track, width < displayWidth ? { width } : undefined); + const separate: TimecodedImage[] = new Array(requested.length); + + let i = 0; + for await (const wrapped of sink.canvasesAtTimestamps(ordered.map(({ time }) => firstTimestamp + time))) { + const { time, index } = ordered[i++]!; + if (!wrapped) throw new DapiError("not-found", `No frame found at ${time}s.`); + const timecode = formatTimecode(time, asset.frameRate); + if (sheets) await sheets.add(index, { at: time, timecode, image: wrapped.canvas }); + else separate[index] = { timecode, png: await encodePng(wrapped.canvas) }; + } + + return sheets ? sheets.result() : separate; + } finally { + input.dispose(); + } +}; + +/** + * A time within the clip. A negative time is an offset back from the end: + * -1 is one second before the end, -1f one frame before it. + */ +function resolveTime(t: number, duration: number): number { + if (t >= 0) { + if (t > duration) { + throw new DapiError("invalid-input", `time ${t}s is past the asset's duration (${duration.toFixed(2)}s).`); + } + return t; + } + const resolved = duration + t; + if (resolved < 0) { + throw new DapiError("invalid-input", `time ${t} counts past the start of the clip (duration ${duration.toFixed(2)}s).`); + } + return resolved; +} diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.ts new file mode 100644 index 00000000..4a5cdd9d --- /dev/null +++ b/apps/web/src/dapi/handlers/media-listen.ts @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { Project, transcodeForAnalysis } from "@diffusionstudio/runtime"; +import { trpc } from "@/lib/trpc"; +import { startResumableSession, uploadResumableStream } from "@/lib/uploads"; +import { requireAssetType, resolveAsset } from "../lib/assets"; + +import type { ToolHandler } from "../handler"; + +export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end, stripVideo }, ctx) => { + ctx.app.requireUser(); + const asset = await resolveAsset(ctx, path); + requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset"); + + const hasWindow = start !== undefined || end !== undefined; + const audioOnly = stripVideo !== false && asset.type === "VIDEO"; + const contentType = asset.type === "VIDEO" && !audioOnly ? "video/mp4" : "audio/ogg"; + + // One upload per distinct analysis input, so a repeated question about the + // same span reuses the transcode. + const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : ""; + const uploadId = `${ctx.session()?.world.get(Project)?.id ?? "project"}-${asset.id}-analyze${audioOnly ? "-audio" : ""}${window}` + .replace(/[^A-Za-z0-9._-]/g, "_"); + const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({ action: "resumable", id: uploadId, contentType }); + + // An upload URL means the server does not have this input yet. + if (uploadUrl) { + const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo: audioOnly }); + const sessionUrl = await startResumableSession(uploadUrl, contentType); + const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl); + await transcoder.run?.(); + await uploadPromise; + } + + const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt }); + return { result: analysis, start, end }; +}; diff --git a/apps/web/src/dapi/handlers/media-probe.ts b/apps/web/src/dapi/handlers/media-probe.ts new file mode 100644 index 00000000..525ddff5 --- /dev/null +++ b/apps/web/src/dapi/handlers/media-probe.ts @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { ALL_FORMATS, BlobSource, Input } from "mediabunny"; +import { getAssetFile } from "@diffusionstudio/runtime"; +import { assetName } from "@diffusionstudio/assets"; +import { resolveAsset } from "../lib/assets"; + +import type { ToolHandler } from "../handler"; + +const PROBE_SAMPLE_PACKETS = 200; + +export const mediaProbe: ToolHandler<"media_probe"> = async ({ path }, ctx) => { + const asset = await resolveAsset(ctx, path); + const blob = await getAssetFile(asset); + const base = { + id: asset.id, + name: assetName(asset), + path: asset.path, + type: asset.type, + mimeType: asset.mimeType, + size: blob.size, + ...("width" in asset && { width: asset.width, height: asset.height }), + }; + + const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) }); + try { + const format = await input.getFormat(); + const mimeType = await input.getMimeType(); + const duration = await input.computeDuration(); + const { images, ...tags } = await input.getMetadataTags(); + delete tags.raw; + + const tracks = []; + for (const track of await input.getTracks()) { + const stats = await track.computePacketStats(PROBE_SAMPLE_PACKETS); + tracks.push({ + id: track.id, + type: track.type, + codec: track.codec, + language: track.languageCode, + firstTimestamp: await track.getFirstTimestamp(), + duration: await track.computeDuration(), + ...stats, + ...(track.isVideoTrack() && { + codedWidth: track.codedWidth, + codedHeight: track.codedHeight, + displayWidth: track.displayWidth, + displayHeight: track.displayHeight, + rotation: track.rotation, + }), + ...(track.isAudioTrack() && { + sampleRate: track.sampleRate, + channels: track.numberOfChannels, + }), + }); + } + + return { + ...base, + format: format.name, + mimeType, + duration, + tags: { ...tags, ...(images?.length && { attachedImages: images.length }) }, + tracks, + }; + } catch { + return { ...base, format: null, tracks: [] }; + } finally { + input.dispose(); + } +}; diff --git a/apps/web/src/dapi/handlers/media-transcribe.ts b/apps/web/src/dapi/handlers/media-transcribe.ts new file mode 100644 index 00000000..234c2632 --- /dev/null +++ b/apps/web/src/dapi/handlers/media-transcribe.ts @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { transcodeForTranscription } from "@diffusionstudio/runtime"; +import { trpc } from "@/lib/trpc"; +import { uploadBlob } from "@/lib/uploads"; +import { requireAssetType, resolveAsset } from "../lib/assets"; + +import type { TranscriptSegment } from "@diffusionstudio/dapi"; +import type { ToolHandler } from "../handler"; + +// Transcripts are remembered per asset for the app's lifetime: the audio +// does not change, and the transcription is the expensive part. +const transcripts = new Map(); + +export const mediaTranscribe: ToolHandler<"media_transcribe"> = async ({ path }, ctx) => { + const asset = await resolveAsset(ctx, path); + requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset"); + + let transcript = transcripts.get(asset.id); + if (!transcript) { + const uploadId = crypto.randomUUID(); + const audioFile = await transcodeForTranscription(asset); + const fileRef = await uploadBlob(audioFile, uploadId); + if (!fileRef) throw new Error(`Failed to upload asset ${asset.id} for transcription.`); + + ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef })); + if (!transcript.length || transcript.every((s) => s.words.length === 0)) { + throw new Error("No speech detected. The audio does not appear to contain recognizable speech."); + } + + transcripts.set(asset.id, transcript); + } + + return { segments: transcript }; +}; diff --git a/apps/web/src/dapi/handlers/media-waveform.ts b/apps/web/src/dapi/handlers/media-waveform.ts new file mode 100644 index 00000000..e15591c8 --- /dev/null +++ b/apps/web/src/dapi/handlers/media-waveform.ts @@ -0,0 +1,15 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { waveformAsset } from "@diffusionstudio/runtime"; +import { resolveAsset } from "../lib/assets"; +import { dataUrlToBytes } from "../lib/png"; + +import type { ToolHandler } from "../handler"; + +export const mediaWaveform: ToolHandler<"media_waveform"> = async ({ path, start, end, scale }, ctx) => { + const asset = await resolveAsset(ctx, path); + const { dataUrl, ...rest } = await waveformAsset(asset, { start, end, scale }); + return { png: dataUrlToBytes(dataUrl), ...rest }; +}; diff --git a/apps/web/src/dapi/handlers/models.ts b/apps/web/src/dapi/handlers/models.ts new file mode 100644 index 00000000..b9d575b9 --- /dev/null +++ b/apps/web/src/dapi/handlers/models.ts @@ -0,0 +1,35 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { + PROMPT_INPUT_IMAGE_MODEL_OPTIONS, + PROMPT_INPUT_VIDEO_MODEL_OPTIONS, + PROMPT_INPUT_AUDIO_MODEL_OPTIONS, +} from "@/components/genai/config"; + +import type { ModelInfo } from "@diffusionstudio/dapi"; +import type { ToolHandler } from "../handler"; + +export const models: ToolHandler<"models"> = async ({ type }) => { + const out: ModelInfo[] = []; + if (!type || type === "image") { + for (const { id, name } of PROMPT_INPUT_IMAGE_MODEL_OPTIONS) out.push({ type: "image", id, name }); + } + if (!type || type === "video") { + for (const option of PROMPT_INPUT_VIDEO_MODEL_OPTIONS) { + out.push({ + type: "video", + id: option.id, + name: option.name, + durations: option.durations, + aspectRatios: option.aspectRatios, + features: option.features, + }); + } + } + if (!type || type === "audio") { + for (const { id, name } of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) out.push({ type: "audio", id, name }); + } + return out; +}; diff --git a/apps/web/src/dapi/handlers/open.ts b/apps/web/src/dapi/handlers/open.ts new file mode 100644 index 00000000..f4c0dcfb --- /dev/null +++ b/apps/web/src/dapi/handlers/open.ts @@ -0,0 +1,7 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { ToolHandler } from "../handler"; + +export const open: ToolHandler<"open"> = ({ dir }, ctx) => ctx.app.openProject(dir); diff --git a/apps/web/src/dapi/handlers/screenshot.ts b/apps/web/src/dapi/handlers/screenshot.ts new file mode 100644 index 00000000..8c9d2c67 --- /dev/null +++ b/apps/web/src/dapi/handlers/screenshot.ts @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mainBridge } from "@/lib/ipc"; +import { MAIN_CHANNELS } from "@desktop/main-channels"; + +import type { ToolHandler } from "../handler"; + +export const screenshot: ToolHandler<"screenshot"> = () => mainBridge.call(MAIN_CHANNELS.WINDOW_CAPTURE, undefined); diff --git a/apps/web/src/context/dapi/voices.ts b/apps/web/src/dapi/handlers/voices.ts similarity index 57% rename from apps/web/src/context/dapi/voices.ts rename to apps/web/src/dapi/handlers/voices.ts index 9109a8d0..e589b0c0 100644 --- a/apps/web/src/context/dapi/voices.ts +++ b/apps/web/src/dapi/handlers/voices.ts @@ -4,12 +4,7 @@ import { PROMPT_INPUT_VOICE_OPTIONS } from "@/components/genai/config"; -export function handleVoices() { - return async () => { - return PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ - id: v.value, - label: v.label, - description: v.description, - })); - } -} +import type { ToolHandler } from "../handler"; + +export const voices: ToolHandler<"voices"> = async () => + PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ id: v.value, label: v.label, description: v.description })); diff --git a/apps/web/src/dapi/handlers/whoami.ts b/apps/web/src/dapi/handlers/whoami.ts new file mode 100644 index 00000000..23022db5 --- /dev/null +++ b/apps/web/src/dapi/handlers/whoami.ts @@ -0,0 +1,7 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { ToolHandler } from "../handler"; + +export const whoami: ToolHandler<"whoami"> = async (_, ctx) => ctx.app.user(); diff --git a/apps/web/src/context/dapi/index.ts b/apps/web/src/dapi/index.ts similarity index 75% rename from apps/web/src/context/dapi/index.ts rename to apps/web/src/dapi/index.ts index b86dc190..3d31fac2 100644 --- a/apps/web/src/context/dapi/index.ts +++ b/apps/web/src/dapi/index.ts @@ -2,4 +2,4 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -export * from './api'; +export { EditorApi, EditorApiProvider, useEditorApi } from "./api"; diff --git a/apps/web/src/dapi/lib/assets.ts b/apps/web/src/dapi/lib/assets.ts new file mode 100644 index 00000000..08e54f3c --- /dev/null +++ b/apps/web/src/dapi/lib/assets.ts @@ -0,0 +1,41 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { getLibrary } from "@diffusionstudio/runtime"; +import { AssetLibrary, isAbsoluteSource, isUrlSource } from "@diffusionstudio/assets"; +import { DapiError } from "@diffusionstudio/dapi"; +import { createProjectFS } from "@/projects/fs"; + +import type { Asset } from "@diffusionstudio/assets"; +import type { ToolContext } from "../handler"; + +/** + * Resolves a tool target by path. With a project open, its library answers: + * library paths look assets up, absolute paths and URLs are described in place + * without being added (transient assets). With none open, a throwaway library + * over a project-less FS describes absolute paths and URLs the same way — a + * fresh one per request, so nothing is remembered between calls. + */ +export function resolveAsset(ctx: ToolContext, path: string): Promise { + const world = ctx.session()?.world; + if (world) return getLibrary(world).resolve(path); + if (!isAbsoluteSource(path) && !isUrlSource(path)) { + throw new DapiError( + "no-project", + `Could not resolve "${path}": with no project open only absolute paths and URLs resolve — open a project to use library paths.`, + ); + } + return new AssetLibrary(createProjectFS("")).resolve(path); +} + +/** Narrows the asset to one of `types`, or throws a `wrong-kind` error naming what was expected. */ +export function requireAssetType( + asset: Asset, + types: readonly T[], + expected: string, +): asserts asset is Extract { + if (!(types as readonly string[]).includes(asset.type)) { + throw new DapiError("wrong-kind", `Asset ${asset.id} is not ${expected}.`); + } +} diff --git a/apps/web/src/context/dapi/frame-triage.ts b/apps/web/src/dapi/lib/frame-triage.ts similarity index 100% rename from apps/web/src/context/dapi/frame-triage.ts rename to apps/web/src/dapi/lib/frame-triage.ts diff --git a/apps/web/src/context/dapi/nodes.ts b/apps/web/src/dapi/lib/nodes.ts similarity index 58% rename from apps/web/src/context/dapi/nodes.ts rename to apps/web/src/dapi/lib/nodes.ts index 690fb67d..e93772df 100644 --- a/apps/web/src/context/dapi/nodes.ts +++ b/apps/web/src/dapi/lib/nodes.ts @@ -4,10 +4,11 @@ import { AdjustmentLayer, Geometry, Group, Scene, Source } from "@diffusionstudio/runtime"; import { parseSource } from "@diffusionstudio/jsx"; +import { DapiError } from "@diffusionstudio/dapi"; import type { Entity, World } from "koota"; -// Scenes are nodes too, so node-targeting endpoints accept them alongside +// Scenes are nodes too, so node-targeting tools accept them alongside // geometry, groups, and adjustment layers. export function isNode(entity: Entity): boolean { return entity.has(Geometry) @@ -16,13 +17,7 @@ export function isNode(entity: Entity): boolean { || entity.has(Scene); } -/** - * The entity `id` names. An id is a source id — the durable name an element - * carries in the project's JSX — not a koota entity number, which is minted - * anew on every recompile and so names nothing across calls. The bare id is - * enough while it names one element; `file:id` settles a tie between files, - * and a positional stamp (`index.tsx:12`) addresses elements that have no id. - */ +/** The one node an id names, or the error that says why there is not exactly one. */ export function resolveNode(world: World, id: string): Entity { const matches = world.query(Source).filter((entity) => { const stamp = entity.get(Source)!.value; @@ -31,20 +26,23 @@ export function resolveNode(world: World, id: string): Entity { return parsed !== undefined && String(parsed.locator) === id; }); if (!matches.length) { - throw new Error(`No such node: "${id}" — node ids are the id attributes in the project's JSX`); + throw new DapiError("not-found", `No such node: "${id}" — node ids are the id attributes in the project's JSX`); } const nodes = matches.filter(isNode); if (!nodes.length) { - throw new Error(`"${id}" is not a node; target a scene, group, clip, or adjustment layer`); + throw new DapiError("wrong-kind", `"${id}" is not a node; target a scene, group, clip, or adjustment layer`); } if (nodes.length > 1) { const stamps = [...new Set(nodes.map((node) => node.get(Source)!.value))]; if (stamps.length === 1) { // One element, many entities: a loop renders its body once per item. - throw new Error(`"${id}" renders ${nodes.length} times (it sits in a loop) — target its scene instead`); + throw new DapiError("ambiguous", `"${id}" renders ${nodes.length} times (it sits in a loop) — target its scene instead`); } - throw new Error(`"${id}" is ambiguous between ${stamps.map((stamp) => `"${stamp}"`).join(", ")} — use the file:id form`); + throw new DapiError( + "ambiguous", + `"${id}" is ambiguous between ${stamps.map((stamp) => `"${stamp}"`).join(", ")} — use the file:id form`, + ); } return nodes[0]!; } diff --git a/apps/web/src/dapi/lib/png.ts b/apps/web/src/dapi/lib/png.ts new file mode 100644 index 00000000..a0973f95 --- /dev/null +++ b/apps/web/src/dapi/lib/png.ts @@ -0,0 +1,9 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** The bytes of a `data:` URL, for previews the runtime renders that way. */ +export function dataUrlToBytes(dataUrl: string): Uint8Array { + const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1); + return Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); +} diff --git a/apps/web/src/dapi/lib/scene.ts b/apps/web/src/dapi/lib/scene.ts new file mode 100644 index 00000000..d62c79fc --- /dev/null +++ b/apps/web/src/dapi/lib/scene.ts @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { getParentNode, isScene, Source } from "@diffusionstudio/runtime"; +import { DapiError } from "@diffusionstudio/dapi"; +import { resolveNode } from "./nodes"; + +import type { Entity, World } from "koota"; + +/** + * The scene an id names. A scene is the unit an export renders, and capture's + * promise is that its frames are an export's frames, so both take scenes only: + * framing an arbitrary node would need its bounds measured across the + * requested positions first, and that pre-roll runs the project's code ahead + * of the frames being drawn, which is exactly what an export never does. + * + * `verb` names the caller in the error ("capture", "export"). + */ +export function requireScene(world: World, id: string, verb: string): Entity { + const node = resolveNode(world, id); + if (isScene(node)) return node; + + let scene = getParentNode(node); + while (scene !== null && !isScene(scene)) scene = getParentNode(scene); + const stamp = scene?.get(Source)?.value; + throw new DapiError( + "wrong-kind", + stamp + ? `"${id}" is not a scene — ${verb} renders what an export renders. ${capitalize(verb)} its scene "${stamp}" instead.` + : `"${id}" is not a scene — ${verb} renders what an export renders, so it takes a scene id.`, + ); +} + +function capitalize(word: string): string { + return word.charAt(0).toUpperCase() + word.slice(1); +} diff --git a/apps/web/src/dapi/lib/sheets.ts b/apps/web/src/dapi/lib/sheets.ts new file mode 100644 index 00000000..76647f3b --- /dev/null +++ b/apps/web/src/dapi/lib/sheets.ts @@ -0,0 +1,87 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { composeSheet, planSheet, planSheetSizes, sheetTimecode } from "@diffusionstudio/encoder"; + +import type { SheetPlan } from "@diffusionstudio/encoder"; +import type { TimecodedImage } from "@diffusionstudio/dapi"; + +export type SheetFrame = { + /** Where the frame sits, in the caller's clock; orders cells within a sheet's label. */ + at: number; + timecode: string; + image: CanvasImageSource; +}; + +/** + * Lays `total` frames out over contact sheets and composes each sheet as + * soon as its last frame arrives, so a long run never holds every frame at + * once. Frames may arrive in any order; sheets come out in order. + * + * Plans are made up front from the frames' source size so the caller can + * decode or render no larger than a cell will be drawn (see `cellWidth`, + * `cellHeight`). + */ +export class SheetCollector { + readonly plans: SheetPlan[]; + /** The largest cell across sheets: nothing needs to be bigger than this. */ + readonly cellWidth: number; + readonly cellHeight: number; + + private readonly sizes: number[]; + private readonly sheetOf: number[] = []; + private readonly firstIndex: number[] = []; + private readonly missing: number[]; + private readonly frames: Array; + private readonly sheets: Array; + + constructor(total: number, source: { width: number; height: number }, perSheet?: number) { + const sizes = planSheetSizes(total, perSheet); + this.sizes = sizes; + this.plans = sizes.map((size) => planSheet(size, source)); + this.cellWidth = Math.max(...this.plans.map((plan) => plan.cellWidth)); + this.cellHeight = Math.max(...this.plans.map((plan) => plan.cellHeight)); + for (const [sheet, size] of sizes.entries()) { + this.firstIndex.push(this.sheetOf.length); + for (let k = 0; k < size; k++) this.sheetOf.push(sheet); + } + this.missing = [...sizes]; + this.frames = new Array(total); + this.sheets = new Array(sizes.length); + } + + /** + * Adds the frame at `index` (its position in the requested order). When it + * completes a sheet, that sheet is composed and its images released: + * ImageBitmaps are closed, and every frame image is dropped from memory. + */ + async add(index: number, frame: SheetFrame): Promise { + this.frames[index] = frame; + const sheet = this.sheetOf[index]!; + if (--this.missing[sheet]! > 0) return; + + const from = this.firstIndex[sheet]!; + const to = from + this.sizes[sheet]!; + const cells = this.frames.slice(from, to).filter((f): f is SheetFrame => f !== undefined); + this.sheets[sheet] = { + timecode: sheetTimecode(cells), + png: await composeSheet( + cells.map((cell) => ({ image: cell.image, label: cell.timecode })), + this.plans[sheet]!, + ), + }; + for (const cell of cells) { + if (cell.image instanceof ImageBitmap) cell.image.close(); + } + this.frames.fill(undefined, from, to); + } + + /** Every sheet, in order. Only complete once every frame has been added. */ + result(): TimecodedImage[] { + return this.sheets.map((sheet, i) => { + if (!sheet) throw new Error(`Contact sheet ${i} is missing frames`); + return sheet; + }); + } +} diff --git a/apps/web/src/context/dapi/session.ts b/apps/web/src/dapi/session.ts similarity index 70% rename from apps/web/src/context/dapi/session.ts rename to apps/web/src/dapi/session.ts index 4b4d1429..8a7819ae 100644 --- a/apps/web/src/context/dapi/session.ts +++ b/apps/web/src/dapi/session.ts @@ -2,13 +2,13 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -// The CLI router mounts once for the app's lifetime (see ./api); what changes -// as projects open and close is this slot. The editor publishes its world and -// project here while it is mounted, and project-bound endpoints read the slot -// per request instead of closing over a world at registration time. +// The tool handlers register once for the app's lifetime (see ./api); what +// changes as projects open and close is this slot. The editor publishes its +// world and project here while it is mounted, and project-bound handlers read +// the slot per request instead of closing over a world at registration time. import { createSignal } from "solid-js"; -import { assert } from "@/utils/common"; +import { DapiError } from "@diffusionstudio/dapi"; import type { Accessor } from "solid-js"; import type { World } from "koota"; @@ -34,9 +34,9 @@ export const editorSession: Accessor = session; /** Set by the editor while a project is open; cleared on its way out. */ export const setEditorSession = setSession; -/** The session, or the failure a CLI caller can act on. */ +/** The session, or the failure a caller can act on. */ export function requireEditorSession(): EditorSession { const current = session(); - assert(current, "No project open — run `dapi open ` first."); + if (!current) throw new DapiError("no-project", "No project open — open one first (`dapi open `)."); return current; } diff --git a/apps/web/src/hooks/use-fullscreen-state.ts b/apps/web/src/hooks/use-fullscreen-state.ts index 8823fcbe..e407d1b1 100644 --- a/apps/web/src/hooks/use-fullscreen-state.ts +++ b/apps/web/src/hooks/use-fullscreen-state.ts @@ -4,7 +4,6 @@ import { createEffect, createResource, onCleanup, onMount } from "solid-js"; -import { handleGetFullscreenState, handleWindowFullscreenChange } from "@/context/dapi/window"; import { mainBridge } from "@/lib/ipc"; import { MAIN_CHANNELS } from "@desktop/main-channels"; @@ -18,9 +17,10 @@ import { MAIN_CHANNELS } from "@desktop/main-channels"; * lives here rather than inside the editor-only API provider. */ export function useFullscreenState() { - const [isFullscreen, { mutate }] = createResource(handleGetFullscreenState, { - initialValue: false, - }); + const [isFullscreen, { mutate }] = createResource( + () => (window.desktop ? mainBridge.call(MAIN_CHANNELS.WINDOW_IS_FULLSCREEN, undefined) : Promise.resolve(false)), + { initialValue: false }, + ); createEffect(() => { document.documentElement.dataset.fullscreen = String(isFullscreen()); @@ -29,10 +29,7 @@ export function useFullscreenState() { onMount(() => { if (!window.desktop) return; onCleanup( - mainBridge.handle( - MAIN_CHANNELS.WINDOW_FULLSCREEN_CHANGE, - handleWindowFullscreenChange(mutate), - ), + mainBridge.handle(MAIN_CHANNELS.WINDOW_FULLSCREEN_CHANGE, ({ fullscreen }) => mutate(fullscreen)), ); }); diff --git a/apps/web/src/lib/cli-rpc.ts b/apps/web/src/lib/cli-rpc.ts deleted file mode 100644 index dbaf1e12..00000000 --- a/apps/web/src/lib/cli-rpc.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { initTRPC } from "@trpc/server"; -import type { AnyTRPCRouter } from "@trpc/server"; -import type { ProcedureCaller, RouterCaller } from "./ipc"; - -// tRPC instance for the CLI-facing routers. Procedures close over their -// dependencies (engine, session), so there is no request context. The -// renderer IS the server here, so opt out of tRPC's browser guard. -export const t = initTRPC.create({ allowOutsideOfServer: true }); - -// Inputs come pre-typed from the tRPC client, so parsers are identity casts -// rather than schemas; validation stays where it always was, in the handlers. -const input = - () => - (value: unknown) => - value as I; - -// Lift an existing unary handler into a procedure, inferring its input and -// output types. The cast collapses tRPC's conditional input type, which -// stays unresolved for a generic I. Zero-argument handlers use q0/m0: they -// skip .input() so the client can call them without an argument. -export const q = (fn: (data: I) => O | Promise) => - t.procedure.input(input()).query(({ input: data }) => fn(data as I)); - -export const m = (fn: (data: I) => O | Promise) => - t.procedure.input(input()).mutation(({ input: data }) => fn(data as I)); - -export const q0 = (fn: () => O | Promise) => t.procedure.query(() => fn()); - -export const m0 = (fn: () => O | Promise) => t.procedure.mutation(() => fn()); - -// Adapts a router for the CLI bridge: resolve a dot-joined procedure path to -// an invocable, or undefined when this router doesn't own the path (the -// bridge then tries other routers or holds the request). -export function createRouterCaller(router: AnyTRPCRouter): RouterCaller { - const caller = t.createCallerFactory(router)({}); - return (path: string): ProcedureCaller | undefined => { - if (!(path in router._def.procedures)) return undefined; - return (data: unknown) => - path.split(".").reduce((o, key) => o[key], caller)(data) as Promise; - }; -} diff --git a/apps/web/src/lib/ipc.ts b/apps/web/src/lib/ipc.ts index 54447eaf..090b6359 100644 --- a/apps/web/src/lib/ipc.ts +++ b/apps/web/src/lib/ipc.ts @@ -12,23 +12,15 @@ import type { MainRequestChannel, MainRequestMap, } from "@desktop/main-channels"; -import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; -import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol"; type EventHandler = (data: MainEventMap[C]) => void; -export type ProcedureCaller = (input: unknown) => Promise; - -// Resolves a dot-joined tRPC procedure path to an invocable, or undefined -// when the router doesn't own that path. -export type RouterCaller = (path: string) => ProcedureCaller | undefined; - type Pending = { resolve: (value: unknown) => void; reject: (error: Error) => void; }; -// Renderer↔main bridge. Symmetric with cliBridge: `handle` registers a +// Renderer↔main bridge: `handle` registers a // single-subscriber receiver for inbound channels (events from main); `call` // sends a request to main and awaits the reply. class MainBridge { @@ -94,87 +86,3 @@ class MainBridge { } export const mainBridge = new MainBridge(); - -type PendingCliRequest = { req: CliRequest; ws: WebSocket }; - -// CLI bridge — answers CLI requests. Each CLI command hosts a short-lived -// WebSocket server; main relays only the connect info (CLI_WIRE.CONNECT) and -// we dial the CLI directly, so payloads never pass through main. One request -// and one reply per connection. The bridge is transport-only: the tRPC -// router registers as a path resolver, and requests arriving before it -// mounts (or between remounts on project switches) are held and retried on -// the next registration — no explicit ready signal needed. -class CliBridge { - // Several routers coexist: the shell router (always mounted, owns `ping` - // and `open`) and the editor router (mounted per open project). Paths are - // disjoint; a request is answered by whichever router owns its path. - private routers = new Set(); - private pending: PendingCliRequest[] = []; - - constructor() { - // Bind eagerly so CONNECT arrivals during page bootstrap are caught - // rather than silently dropped before any router registers. - if (window.desktop) { - window.desktop.on(CLI_WIRE.CONNECT, (payload) => { - const { port, token } = payload as CliHandshake; - const ws = new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`); - ws.onmessage = (event) => { - try { - const req = JSON.parse(event.data as string) as CliRequest; - void this.dispatch({ req, ws }); - } catch (err) { - console.error("[cli-bridge] malformed CLI request", err); - ws.close(); - } - }; - }); - } - } - - private resolve(path: string): ProcedureCaller | undefined { - for (const router of this.routers) { - const proc = router(path); - if (proc) return proc; - } - return undefined; - } - - private async dispatch(pending: PendingCliRequest): Promise { - const { req, ws } = pending; - const proc = this.resolve(req.path); - if (!proc) { - this.pending.push(pending); - return; - } - let reply: CliReply; - try { - const data = await proc(req.input); - reply = { ok: true, data }; - } catch (err) { - reply = { ok: false, error: (err as Error).message }; - } - try { - ws.send(JSON.stringify(reply)); - } catch (err) { - ws.send( - JSON.stringify({ - ok: false, - error: `Failed to serialize reply for ${req.path}: ${(err as Error).message}`, - }), - ); - } - } - - register(router: RouterCaller): () => void { - if (!window.desktop) return () => {}; - this.routers.add(router); - const held = this.pending; - this.pending = []; - for (const pending of held) void this.dispatch(pending); - return () => { - this.routers.delete(router); - }; - } -} - -export const cliBridge = new CliBridge(); diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsx index 81e054ef..1c3d9437 100644 --- a/apps/web/src/pages/editor.tsx +++ b/apps/web/src/pages/editor.tsx @@ -8,7 +8,7 @@ import { Timeline, Layers } from "@/components/timeline"; import { Soundboard, Inspector } from "@/components/sidebar-right"; import { FloatingProjectHeader, SidebarLeft } from "@/components/sidebar-left"; import { useLayout, MIN_TIMELINE_HEIGHT } from "@/context/layout"; -import { useEditorApi } from "@/context/dapi"; +import { useEditorApi } from "@/dapi"; import { RULER_HEIGHT } from "@/engine/timeline"; import { createEffect, onCleanup, untrack } from 'solid-js'; import { toast } from 'somoto'; diff --git a/apps/web/src/pages/project.tsx b/apps/web/src/pages/project.tsx index f6cf3ada..f0ff44ca 100644 --- a/apps/web/src/pages/project.tsx +++ b/apps/web/src/pages/project.tsx @@ -7,7 +7,7 @@ import { Navigate, useNavigate } from '@solidjs/router'; import { EditorPage } from './editor'; import { LayoutProvider } from "@/context/layout"; import { PromptInputProvider } from "@/context/prompt-input"; -import { EditorApiProvider } from '@/context/dapi'; +import { EditorApiProvider } from '@/dapi'; import { ExportProvider } from '@/context/export'; import { ProjectProvider } from '@/context/project'; import { projectRoute, useProjectRef } from '@/hooks/use-project-route'; diff --git a/docs/mcp-server.md b/docs/mcp-server.md index 5b93ba17..d32644b3 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -193,7 +193,7 @@ Removed from the CLI package: `cli-client.ts`, `cli-channels.ts`, the tRPC and ` The CLI ships inside the app bundle, so there is no compatibility window to maintain between old CLIs and new apps. This can land as one feature branch in ordered steps, each of which builds and type-checks on its own. 1. **Catalog.** Create `packages/dapi` with the tool catalog, `Time`, and `DapiError`, plus tests. Point the renderer handlers' request and result types at it. Delete `apps/cli/src/cli-channels.ts`; `apps/cli/src/protocol.ts` keeps only the handshake and envelope types of the current transport. No behaviour change. *Landed on `mcp-support`.* -2. **Renderer.** Replace the tRPC router in `api.tsx` with a handler map keyed by tool name, validated by the catalog schemas, with `AbortSignal` and progress plumbed in. Replace the WebSocket half of `CliBridge` with the `dapi:call` IPC handling. At this step the old CLI stops working; that is expected. +2. **Renderer.** Replace the tRPC router in `api.tsx` with one handler per tool behind a uniform `(args, ctx)` signature, validated by the catalog schemas, with an `AbortSignal` per call. Handlers return bytes; the WebSocket transport carries them as tagged base64 until step 3 replaces it. The CLI drops tRPC for a typed `call(name, input)` over the same catalog, so it keeps working at every step and its import of `apps/web` is gone. Progress reporting waits for step 3, with the transport that carries it. *Landed on `mcp-support`.* 3. **Main.** Add `mcp-server.ts` with the socket transport, catalog registration, the in-flight map, result presentation, and the `logs`, `fonts`, `fetch` and `report` handlers. Delete `cli-server.ts`. Verify with the MCP Inspector against the socket. 4. **CLI.** Rewrite `apps/cli` as `mcp`, `open`, `call` and the wrappers. Drop tRPC and `ws`. Update `cli-install.ts` and the settings UI to offer MCP registration. 5. **Docs.** Update the skill and regenerate `reference/`. Update the installation reference the skill points to. diff --git a/package-lock.json b/package-lock.json index f16d06a3..23f685b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,8 +25,6 @@ "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", "@diffusionstudio/dapi": "*", - "@trpc/client": "^11.18.0", - "@trpc/server": "^11.18.0", "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", "esbuild": "^0.28.1", diff --git a/packages/dapi/src/catalog.test.ts b/packages/dapi/src/catalog.test.ts index cba7fd8e..c0641573 100644 --- a/packages/dapi/src/catalog.test.ts +++ b/packages/dapi/src/catalog.test.ts @@ -26,10 +26,10 @@ describe("catalog", () => { } }); - it("converts every schema to JSON Schema without throwing", () => { + it("converts every schema to JSON Schema without throwing (bytes have no JSON form and pass as any)", () => { for (const tool of catalog) { expect(() => z.toJSONSchema(tool.input, { io: "input" }), `${tool.name} input`).not.toThrow(); - expect(() => z.toJSONSchema(tool.output), `${tool.name} output`).not.toThrow(); + expect(() => z.toJSONSchema(tool.output, { unrepresentable: "any" }), `${tool.name} output`).not.toThrow(); } }); diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts index d6ab997b..da23f6c8 100644 --- a/packages/dapi/src/index.ts +++ b/packages/dapi/src/index.ts @@ -19,7 +19,8 @@ export type { TimeInput } from "./time"; export { DapiError, isDapiError } from "./errors"; export type { DapiErrorCode } from "./errors"; -export { MAX_FRAMES_PER_SHEET } from "./schemas"; +export { MAX_FRAMES_PER_SHEET, Bytes } from "./schemas"; +export { parseToolArgs } from "./validate"; export { FRAME_CAP } from "./tools/media-grab"; export { ISSUE_LOG_TAIL } from "./tools/report"; diff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.ts index 22ffad93..7557a8c7 100644 --- a/packages/dapi/src/schemas.ts +++ b/packages/dapi/src/schemas.ts @@ -29,12 +29,18 @@ export const AssetPath = z export const MAX_FRAMES_PER_SHEET = 12; /** - * One written image: a single frame stamped with its timecode, or a contact - * sheet stamped with the span it covers (`0f-08s10f`). + * Raw bytes as a handler produces them. How they travel is the transport's + * business: base64 on a JSON wire, structured clone over IPC, a file on disk. + */ +export const Bytes = z.custom((value) => value instanceof Uint8Array, "expected bytes (a Uint8Array)"); + +/** + * One image: a single frame stamped with its timecode, or a contact sheet + * stamped with the span it covers (`0f-08s10f`). */ export const TimecodedImage = z.object({ timecode: z.string(), - base64: z.string().describe("PNG bytes, base64"), + png: Bytes, }); /** diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.ts index c6b40ba8..9c30c007 100644 --- a/packages/dapi/src/tools/context.ts +++ b/packages/dapi/src/tools/context.ts @@ -32,11 +32,11 @@ export const context = defineTool({ input: z.object({}), output: z.union([ z.object({ - rootDir: z.string().describe("folder projects live under"), + rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"), projectDir: z.null(), }), z.object({ - rootDir: z.string().describe("folder projects live under"), + rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"), projectDir: z.string().describe("absolute path of the open project"), currentTime: z .number() diff --git a/packages/dapi/src/tools/media-filmstrip.ts b/packages/dapi/src/tools/media-filmstrip.ts index 1de24cc2..dd9cdd1c 100644 --- a/packages/dapi/src/tools/media-filmstrip.ts +++ b/packages/dapi/src/tools/media-filmstrip.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { defineTool } from "../tool"; -import { AssetPath, checkWindow, windowFields } from "../schemas"; +import { AssetPath, Bytes, checkWindow, windowFields } from "../schemas"; /** The window and scale that filmstrip and waveform share. */ export const previewFields = { @@ -22,6 +22,6 @@ export const mediaFilmstrip = defineTool({ description: "Render a grid of thumbnails sampled across the timeline to a PNG (local render, no credits), each row stamped with an HH:MM:SS:FF ruler. A fast, token-efficient video track preview; narrow the window to zoom into a region of interest. Video only (use media_waveform for audio).", input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), - output: z.looseObject({ base64: z.string().describe("PNG bytes, base64") }), + output: z.looseObject({ png: Bytes }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/media-waveform.ts b/packages/dapi/src/tools/media-waveform.ts index 61a17c45..d0691d5d 100644 --- a/packages/dapi/src/tools/media-waveform.ts +++ b/packages/dapi/src/tools/media-waveform.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { defineTool } from "../tool"; -import { AssetPath, checkWindow } from "../schemas"; +import { AssetPath, Bytes, checkWindow } from "../schemas"; import { previewFields } from "./media-filmstrip"; export const mediaWaveform = defineTool({ @@ -14,7 +14,7 @@ export const mediaWaveform = defineTool({ "Render the audio track of a video or audio file as a waveform PNG (local render, no credits) with a timestamp ruler: loudness over time, with silent stretches highlighted in red. A fast, token-efficient audio track preview; the silent spans are also returned as second ranges.", input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), output: z.looseObject({ - base64: z.string().describe("PNG bytes, base64"), + png: Bytes, silences: z.array(z.object({ start: z.number(), end: z.number() })).describe("seconds"), }), runsIn: "renderer", diff --git a/packages/dapi/src/tools/screenshot.ts b/packages/dapi/src/tools/screenshot.ts index c5f7b5ed..9f54bd58 100644 --- a/packages/dapi/src/tools/screenshot.ts +++ b/packages/dapi/src/tools/screenshot.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { defineTool } from "../tool"; +import { Bytes } from "../schemas"; export const screenshot = defineTool({ name: "screenshot", @@ -12,7 +13,7 @@ export const screenshot = defineTool({ "Capture the entire application window as a PNG — the full UI as the user sees it (panels, timeline, asset library, canvas viewport), at the window's current size. The tool for checking what the app itself looks like; to render a node or scene cleanly for composition checks use capture instead.", input: z.object({}), output: z.object({ - base64: z.string().describe("PNG bytes, base64"), + png: Bytes, width: z.number(), height: z.number(), }), diff --git a/packages/dapi/src/tools/whoami.ts b/packages/dapi/src/tools/whoami.ts index d1ba3f9d..f2d5cf0b 100644 --- a/packages/dapi/src/tools/whoami.ts +++ b/packages/dapi/src/tools/whoami.ts @@ -10,6 +10,6 @@ export const whoami = defineTool({ title: "Signed-in account", description: "Report the authenticated account, or null if signed out.", input: z.object({}), - output: z.looseObject({ id: z.string(), email: z.string().optional() }).nullable(), + output: z.object({ id: z.string(), email: z.string().optional() }).nullable(), runsIn: "renderer", }); diff --git a/packages/dapi/src/validate.test.ts b/packages/dapi/src/validate.test.ts new file mode 100644 index 00000000..6a14b389 --- /dev/null +++ b/packages/dapi/src/validate.test.ts @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from "vitest"; +import { isDapiError } from "./errors"; +import { parseToolArgs } from "./validate"; + +describe("parseToolArgs", () => { + it("returns parsed arguments with defaults applied", () => { + expect(parseToolArgs("capture", { id: "intro" })).toEqual({ id: "intro", combine: true }); + }); + + it("throws an invalid-input DapiError that names every bad field on one line", () => { + try { + parseToolArgs("media_grab", { path: "/c.mp4", times: ["abc"], count: 0 }); + } catch (e) { + expect(isDapiError(e) && e.code).toBe("invalid-input"); + const message = (e as Error).message; + expect(message).toMatch(/^Invalid arguments for media_grab: /); + expect(message).toMatch(/times\.0: expected a time/); + expect(message).toMatch(/count: /); + expect(message).not.toContain("\n"); + return; + } + throw new Error("expected a throw"); + }); +}); diff --git a/packages/dapi/src/validate.ts b/packages/dapi/src/validate.ts new file mode 100644 index 00000000..89c5e776 --- /dev/null +++ b/packages/dapi/src/validate.ts @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { z } from "zod"; +import { toolByName } from "./catalog"; +import type { ToolArgs, ToolName } from "./catalog"; +import { DapiError } from "./errors"; + +/** + * Validates a tool's raw arguments against its schema. Every issue is + * reported on one line, keyed by the field it points at, so an agent can + * correct the call from the message alone. + */ +export function parseToolArgs(name: N, raw: unknown): ToolArgs { + const result = toolByName(name).input.safeParse(raw); + if (result.success) return result.data as ToolArgs; + throw new DapiError("invalid-input", `Invalid arguments for ${name}: ${formatIssues(result.error.issues)}`, { + cause: result.error, + }); +} + +function formatIssues(issues: readonly z.core.$ZodIssue[]): string { + return issues.map((issue) => `${issue.path.map(String).join(".") || "input"}: ${issue.message}`).join("; "); +} diff --git a/packages/encoder/src/contact-sheet.ts b/packages/encoder/src/contact-sheet.ts index 37259a8b..ac14ed39 100644 --- a/packages/encoder/src/contact-sheet.ts +++ b/packages/encoder/src/contact-sheet.ts @@ -3,6 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { stampTimestampLabel } from '@diffusionstudio/runtime'; +import { encodePng } from './png'; /** * Contact sheets: a handful of frames merged into one labelled image, so a @@ -112,7 +113,7 @@ export function planSheetSizes(total: number, perSheet?: number): number[] { export async function composeSheet( frames: Array<{ image: CanvasImageSource; label: string }>, plan: SheetPlan, -): Promise { +): Promise { const canvas = new OffscreenCanvas(plan.width, plan.height); const ctx = canvas.getContext('2d')!; @@ -133,17 +134,5 @@ export async function composeSheet( ctx.restore(); } - const blob = await canvas.convertToBlob({ type: 'image/png' }); - const dataUrl = await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(reader.result as string); - reader.onerror = () => reject(reader.error); - reader.readAsDataURL(blob); - }); - return dataUrl.slice(dataUrl.indexOf(',') + 1); -} - -export async function decodePngBase64(base64: string): Promise { - const bytes = Uint8Array.from(atob(base64), (c) => c.charCodeAt(0)); - return createImageBitmap(new Blob([bytes], { type: 'image/png' })); + return encodePng(canvas); } diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.ts index c0bf0f48..03919812 100644 --- a/packages/encoder/src/image-encoder.ts +++ b/packages/encoder/src/image-encoder.ts @@ -11,12 +11,13 @@ import { import { captureScene, normalizeSceneTransform, resolverSystem, warmupAssets } from './encoder'; import { scaleSize } from './utils'; +import { encodePng } from './png'; import type { World } from 'koota'; import type { ImageEncoderConfig } from './interfaces'; -/** One capture: the PNG plus the timecode of the frame rendered, e.g. `01s15f`. */ -export type CapturedImage = { base64: string; timecode: string }; +/** One capture: the PNG bytes plus the timecode of the frame rendered, e.g. `01s15f`. */ +export type CapturedImage = { png: Uint8Array; timecode: string }; export type ImageExportResult = | { type: 'success'; data: CapturedImage[] } @@ -25,7 +26,7 @@ export type ImageExportResult = /** * Renders single frames of the scene a capture world holds into standalone - * PNGs (base64, no data-url prefix). + * PNGs. * * `world` is the caller's, built for this capture and holding the scene as * the stage's only child — the same arrangement `createEncoder` takes, and @@ -125,7 +126,7 @@ export async function createImageEncoder(world: World, config: ImageEncoderConfi renderSystem(world); images.set(frame, { - base64: await toBase64Png(canvas), + png: await encodePng(canvas), timecode: formatTimecode(playheadSeconds, frameRate), }); } @@ -148,15 +149,3 @@ export async function createImageEncoder(world: World, config: ImageEncoderConfi }; } -async function toBase64Png(canvas: HTMLCanvasElement): Promise { - const blob = await new Promise((resolve, reject) => { - canvas.toBlob((value) => value ? resolve(value) : reject(new Error('Could not encode PNG')), 'image/png'); - }); - const dataUrl = await new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => resolve(reader.result as string); - reader.onerror = () => reject(reader.error); - reader.readAsDataURL(blob); - }); - return dataUrl.split(',')[1] ?? ''; -} diff --git a/packages/encoder/src/index.ts b/packages/encoder/src/index.ts index ab8b77dd..433dd698 100644 --- a/packages/encoder/src/index.ts +++ b/packages/encoder/src/index.ts @@ -8,5 +8,6 @@ export * from './encoder'; export * from './format'; export * from './image-encoder'; export * from './interfaces'; +export * from './png'; export * from './types'; export * from './utils'; diff --git a/packages/encoder/src/png.ts b/packages/encoder/src/png.ts new file mode 100644 index 00000000..905b6190 --- /dev/null +++ b/packages/encoder/src/png.ts @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// PNG in and out of a canvas, as bytes. Transports decide how bytes travel +// (base64 on a JSON wire, structured clone over IPC, a file on disk); nothing +// in the render path should have to know. + +export async function encodePng(canvas: HTMLCanvasElement | OffscreenCanvas): Promise { + const blob = + canvas instanceof OffscreenCanvas + ? await canvas.convertToBlob({ type: 'image/png' }) + : await new Promise((resolve, reject) => { + canvas.toBlob((value) => (value ? resolve(value) : reject(new Error('Could not encode PNG'))), 'image/png'); + }); + return new Uint8Array(await blob.arrayBuffer()); +} + +export function decodePng(png: Uint8Array): Promise { + return createImageBitmap(new Blob([png as BlobPart], { type: 'image/png' })); +} From 059e1f6e765a26df534e3b9eed2e8616bc9c640e Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Thu, 3 Sep 2026 15:53:35 +0200 Subject: [PATCH 03/13] Update dependencies and refactor CLI integration - Added @modelcontextprotocol/sdk as a dependency in package-lock.json and apps/cli/package.json. - Removed unused @types/ws dependency from package-lock.json and apps/cli/package.json. - Refactored cli-client.ts to utilize the new SDK for tool interactions, enhancing modularity and performance. - Updated media handling functions in index.ts to streamline output handling and improve clarity. - Deleted obsolete protocol.ts and ytdlp.ts files, consolidating functionality within the new SDK. - Introduced edit-types.ts to define types for source edits, improving type safety and maintainability in the editing process. - Updated package.json files to include vitest for testing support in both CLI and desktop applications. --- apps/cli/package.json | 12 +- apps/cli/src/cli-client.ts | 170 +-- apps/cli/src/index.ts | 165 +-- apps/cli/src/protocol.ts | 73 -- apps/cli/src/ytdlp.ts | 74 -- apps/desktop/package.json | 12 +- apps/desktop/scripts/stage-cli.mjs | 5 +- apps/desktop/src/cli-server.ts | 161 --- apps/desktop/src/dapi/handler.ts | 22 + apps/desktop/src/dapi/handlers/fetch.ts | 81 ++ .../src/dapi/handlers}/fonts.ts | 53 +- apps/desktop/src/dapi/handlers/index.ts | 13 + apps/desktop/src/dapi/handlers/logs.ts | 27 + .../src/dapi/handlers}/report.ts | 50 +- apps/desktop/src/dapi/present.test.ts | 68 ++ apps/desktop/src/dapi/present.ts | 110 ++ apps/desktop/src/dapi/renderer-calls.ts | 110 ++ apps/desktop/src/dapi/server.ts | 122 +++ apps/desktop/src/edit-types.ts | 140 +++ apps/desktop/src/edit.ts | 145 +-- apps/desktop/src/headless.ts | 18 + apps/desktop/src/main-channels.ts | 6 +- apps/desktop/src/main.ts | 20 +- apps/desktop/src/preload.ts | 7 +- apps/desktop/src/projects.ts | 2 +- apps/web/package.json | 1 - apps/web/src/context/render.ts | 10 + apps/web/src/dapi/api.tsx | 8 +- apps/web/src/dapi/bridge.ts | 85 +- apps/web/src/dapi/handler.ts | 10 +- apps/web/src/dapi/handlers/context.ts | 2 +- apps/web/src/dapi/handlers/index.ts | 2 - apps/web/src/dapi/handlers/logs.ts | 22 - apps/web/src/dapi/handlers/models.ts | 2 +- apps/web/src/dapi/handlers/voices.ts | 5 +- apps/web/src/dapi/handlers/whoami.ts | 2 +- docs/mcp-server.md | 17 +- package-lock.json | 987 ++++++++++++++++-- packages/dapi/package.json | 1 + packages/dapi/src/catalog.test.ts | 5 +- packages/dapi/src/catalog.ts | 15 +- packages/dapi/src/index.ts | 16 +- packages/dapi/src/ipc.ts | 27 + packages/dapi/src/schemas.ts | 15 +- packages/dapi/src/socket.test.ts | 111 ++ packages/dapi/src/socket.ts | 85 +- packages/dapi/src/tool.ts | 22 +- packages/dapi/src/tools/capture.ts | 6 +- packages/dapi/src/tools/context.ts | 30 +- packages/dapi/src/tools/fonts.ts | 2 +- packages/dapi/src/tools/logs.ts | 2 +- packages/dapi/src/tools/media-filmstrip.ts | 4 +- packages/dapi/src/tools/media-grab.ts | 6 +- packages/dapi/src/tools/media-waveform.ts | 7 +- packages/dapi/src/tools/models.ts | 2 +- packages/dapi/src/tools/screenshot.ts | 7 +- packages/dapi/src/tools/tools.test.ts | 14 +- packages/dapi/src/tools/voices.ts | 2 +- packages/dapi/src/tools/whoami.ts | 4 +- packages/dapi/src/validate.test.ts | 28 - packages/dapi/src/validate.ts | 25 - 61 files changed, 2216 insertions(+), 1039 deletions(-) delete mode 100644 apps/cli/src/protocol.ts delete mode 100644 apps/cli/src/ytdlp.ts delete mode 100644 apps/desktop/src/cli-server.ts create mode 100644 apps/desktop/src/dapi/handler.ts create mode 100644 apps/desktop/src/dapi/handlers/fetch.ts rename apps/{cli/src => desktop/src/dapi/handlers}/fonts.ts (66%) create mode 100644 apps/desktop/src/dapi/handlers/index.ts create mode 100644 apps/desktop/src/dapi/handlers/logs.ts rename apps/{cli/src => desktop/src/dapi/handlers}/report.ts (58%) create mode 100644 apps/desktop/src/dapi/present.test.ts create mode 100644 apps/desktop/src/dapi/present.ts create mode 100644 apps/desktop/src/dapi/renderer-calls.ts create mode 100644 apps/desktop/src/dapi/server.ts create mode 100644 apps/desktop/src/edit-types.ts create mode 100644 apps/desktop/src/headless.ts delete mode 100644 apps/web/src/dapi/handlers/logs.ts create mode 100644 packages/dapi/src/ipc.ts create mode 100644 packages/dapi/src/socket.test.ts delete mode 100644 packages/dapi/src/validate.test.ts delete mode 100644 packages/dapi/src/validate.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index 20c2302e..d708d36a 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -6,11 +6,8 @@ "bin": { "dapi": "./dist/index.js" }, - "exports": { - "./protocol": "./src/protocol.ts" - }, "scripts": { - "build": "esbuild src/index.ts --bundle --platform=node --format=cjs --external:esbuild --external:@babel/core --external:@babel/preset-typescript --external:babel-preset-solid --external:bufferutil --external:utf-8-validate --outfile=dist/index.js && chmod +x dist/index.js", + "build": "esbuild src/index.ts --bundle --platform=node --format=cjs --external:esbuild --external:@babel/core --external:@babel/preset-typescript --external:babel-preset-solid --outfile=dist/index.js && chmod +x dist/index.js", "check": "tsc --noEmit", "symlink:remove": "rm -f /opt/homebrew/bin/dapi", "symlink:create": "npm run build && ln -sf \"$PWD/dist/index.js\" /opt/homebrew/bin/dapi" @@ -18,16 +15,15 @@ "dependencies": { "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", + "@diffusionstudio/dapi": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", - "esbuild": "^0.28.1", - "ws": "^8.18.3", - "@diffusionstudio/dapi": "*" + "esbuild": "^0.28.1" }, "devDependencies": { "@types/babel__core": "^7.20.5", "@types/node": "^24.10.1", - "@types/ws": "^8.18.1", "typescript": "~5.9.3" } } diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts index 0b1a5726..bd3189ff 100644 --- a/apps/cli/src/cli-client.ts +++ b/apps/cli/src/cli-client.ts @@ -3,14 +3,12 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { connect } from "node:net"; -import { randomBytes } from "node:crypto"; -import type { AddressInfo } from "node:net"; -import { WebSocketServer } from "ws"; -import { SOCKET_PATH } from "@diffusionstudio/dapi/socket"; -import { decodeReply } from "./protocol"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { SOCKET_PATH, SocketTransport } from "@diffusionstudio/dapi/socket"; +import { version } from "../../../package.json"; -import type { ToolInput, ToolName, ToolResult } from "@diffusionstudio/dapi"; -import type { CliHandshake, CliHandshakeReply, CliReply, CliRequest } from "./protocol"; +import type { Socket } from "node:net"; +import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi"; const DEFAULT_TIMEOUT_MS = 60000; export const GENERATE_TIMEOUT_MS = 600000; @@ -19,131 +17,73 @@ export const EXPORT_TIMEOUT_MS = 3600000; export type CallOptions = { timeoutMs?: number }; /** - * Calls one tool in the running app. Typed by the catalog: the input is what - * the tool's schema accepts, the result what its handler returns (bytes come - * back as bytes; the wire's base64 is decoded here). + * Calls one tool in the running app over an MCP session on its socket. + * Typed by the catalog: the input is what the tool's schema accepts, the + * output its structured content. One session per call; a command makes one + * or two, and the process exits when it settles. */ export async function call( name: N, input: ToolInput, options: CallOptions = {}, -): Promise> { - return (await transport({ path: name, input }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS)) as ToolResult; -} - -/** Liveness: answered by the renderer's transport before any handler registers. */ -export async function ping(): Promise { - await transport({ path: "ping", input: undefined }, DEFAULT_TIMEOUT_MS); -} - -// Asks the app (via the unix socket) to have the renderer dial our WebSocket -// server. Main replies once the connect info has been delivered, so a -// rejection here means the app is down or the renderer never became ready. -function requestConnection(handshake: CliHandshake, timeoutMs: number): Promise { - return new Promise((resolve, reject) => { - const sock = connect(SOCKET_PATH); - let buf = ""; - let settled = false; - const settle = (fn: () => void) => { - if (settled) return; - settled = true; - sock.destroy(); - fn(); - }; - - sock.setEncoding("utf8"); - sock.setTimeout(timeoutMs, () => - settle(() => reject(new Error("Timed out waiting for the app to accept the connection"))), +): Promise> { + return withClient(async (client) => { + const result = await client.callTool( + { name, arguments: input as Record }, + undefined, + { timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS }, ); - sock.on("connect", () => sock.end(JSON.stringify(handshake))); - sock.on("data", (chunk) => { - buf += chunk; - }); - sock.on("end", () => { - let reply: CliHandshakeReply; - try { - reply = JSON.parse(buf) as CliHandshakeReply; - } catch (e) { - settle(() => reject(e instanceof Error ? e : new Error(String(e)))); - return; - } - if (reply.ok) settle(resolve); - else settle(() => reject(new Error(reply.error))); - }); - sock.on("error", (err) => settle(() => reject(err))); + if (result.isError) { + const text = (result.content as Array<{ type: string; text?: string }>) + .filter((block) => block.type === "text") + .map((block) => block.text) + .join("\n"); + throw new Error(text || `${name} failed`); + } + return result.structuredContent as ToolOutput; }); } -// Each call runs over its own short-lived WebSocket server that the renderer -// dials in to. No cancellation: the CLI process exits when the command settles. -async function transport(request: CliRequest, timeoutMs: number): Promise { - const token = randomBytes(16).toString("hex"); - // Frame batches can exceed ws's 100 MiB default, so disable the payload - // cap; the server only lives for one request. - const wss = new WebSocketServer({ host: "127.0.0.1", port: 0, maxPayload: 0 }); +/** Liveness: a round-trip through the app's MCP server. */ +export function ping(): Promise { + return withClient(async (client) => { + await client.ping(); + }); +} +async function withClient(fn: (client: Client) => Promise): Promise { + const socket = await openSocket(); + const client = new Client({ name: "dapi", version }); try { - const reply = await new Promise((resolve, reject) => { - let settled = false; - const timer = setTimeout(() => { - settle(() => reject(new Error("Timed out waiting for response"))); - }, timeoutMs); - const settle = (fn: () => void) => { - if (settled) return; - settled = true; - clearTimeout(timer); - fn(); - }; - - wss.on("error", (err) => settle(() => reject(err))); - wss.on("connection", (ws, req) => { - const url = new URL(req.url ?? "/", "ws://127.0.0.1"); - if (url.searchParams.get("token") !== token) { - ws.terminate(); - return; - } - ws.on("message", (raw) => { - try { - const parsed = decodeReply(raw.toString()); - settle(() => resolve(parsed)); - } catch (e) { - settle(() => reject(e instanceof Error ? e : new Error(String(e)))); - } - }); - ws.on("close", () => settle(() => reject(new Error("App disconnected before replying")))); - ws.on("error", (err) => settle(() => reject(err))); - ws.send(JSON.stringify(request)); - }); - - wss.once("listening", () => { - const { port } = wss.address() as AddressInfo; - requestConnection({ port, token }, timeoutMs).catch((err) => - settle(() => reject(err instanceof Error ? err : new Error(String(err)))), - ); - }); - }); - - if (reply.ok) return reply.data; - throw new Error(reply.error); + await client.connect(new SocketTransport(socket)); + return await fn(client); } finally { - // Tear down explicitly so no lingering handles keep the Node event loop - // alive past `console.log(result)` and block the CLI from exiting. - for (const client of wss.clients) client.terminate(); - wss.close(); + await client.close().catch(() => {}); } } -/** The errno of a transport failure (ENOENT/ECONNREFUSED when the app is down), if any. */ +// Connecting is where "the app is not running" shows up, as ENOENT (no +// socket file) or ECONNREFUSED (a stale one); see `errnoCode`. +function openSocket(): Promise { + return new Promise((resolve, reject) => { + const socket = connect(SOCKET_PATH); + socket.once("connect", () => { + socket.off("error", reject); + resolve(socket); + }); + socket.once("error", reject); + }); +} + +/** The errno of a connection failure (ENOENT/ECONNREFUSED when the app is down), if any. */ export function errnoCode(e: unknown): string | undefined { return (e as NodeJS.ErrnoException | undefined)?.code; } -// Bridges the cold-start gap after launching the app. Main only delivers the -// handshake once the renderer has finished loading, and `ping` is answered by -// the renderer's transport, so a single round-trip proves the app is fully -// up. The retry loop only handles the brief window before the handshake -// socket itself binds (ENOENT/ECONNREFUSED). -export async function waitForCliSocket(timeoutMs = 30000): Promise { +// Bridges the cold-start gap after launching the app: the socket appears +// once main is ready, and `ping` proves the server answers. The retry loop +// only handles the brief window before the socket binds. +export async function waitForApp(timeoutMs = 30000): Promise { const start = Date.now(); let lastError: unknown = null; while (Date.now() - start < timeoutMs) { @@ -157,7 +97,5 @@ export async function waitForCliSocket(timeoutMs = 30000): Promise { await new Promise((r) => setTimeout(r, 200)); } } - throw lastError instanceof Error - ? lastError - : new Error("Timed out waiting for the app to start"); + throw lastError instanceof Error ? lastError : new Error("Timed out waiting for the app to start"); } diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index b5160bb3..4058434e 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -4,19 +4,14 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import { execFile } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, isAbsolute, join, resolve } from "node:path"; +import { existsSync } from "node:fs"; +import { isAbsolute, resolve } from "node:path"; import { Command } from "commander"; import { version } from "../../../package.json"; import { parseTime, TIME_FPS } from "@diffusionstudio/jsx"; -import { call, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, ping, waitForCliSocket } from "./cli-client"; -import { listLocalFonts } from "./fonts"; -import { buildIssueBody, createIssue } from "./report"; -import { fetchVideo } from "./ytdlp"; +import { call, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, ping, waitForApp } from "./cli-client"; import { ISSUE_LOG_TAIL, MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi"; -import type { FrameQuality, LogEntry, LogLevel, TimecodedImage } from "@diffusionstudio/dapi"; +import type { FrameQuality, ImageRef, LogEntry, LogLevel } from "@diffusionstudio/dapi"; // Long-running commands (renders, AI generation) override the default 60s. const GENERATE = { timeoutMs: GENERATE_TIMEOUT_MS }; @@ -104,10 +99,8 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { const perSheet = parsePerSheet(opts.perSheet, opts.separate); const target = resolveAssetRef(ref); - const dir = opts.output ?? join(tmpdir(), `dapi-grab-${randomUUID().slice(0, 8)}`); - mkdirSync(dir, { recursive: true }); try { - const images = await call("media_grab", { + const { images } = await call("media_grab", { ...target, times, count, @@ -118,8 +111,9 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { combine: !opts.separate, perSheet, uncapped: opts.uncapped, + output: resolveOutput(opts.output), }); - writeImages(images, dir); + printImages(images); } catch (e) { handleSocketError(e); } @@ -205,15 +199,15 @@ function parseTimeArg(value: string, flag: string, allowNegative = false): numbe return seconds; } -// Frames and contact sheets arrive in the same shape: the app stamps each -// image with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the -// filename too. -function writeImages(images: TimecodedImage[], dir: string): void { - for (const { timecode, png } of images) { - const path = join(dir, `${timecode}.png`); - writeFileSync(path, png); - console.log(JSON.stringify({ timecode, path })); - } +// The app wrote the PNGs (one per frame or contact sheet, named by +// timecode); print where each landed, one line per image. +function printImages(images: ImageRef[]): void { + for (const image of images) console.log(JSON.stringify(image)); +} + +/** An output path as the app needs it: absolute, or absent for the app's default. */ +function resolveOutput(path: string | undefined): string | undefined { + return path === undefined ? undefined : resolve(process.cwd(), path); } function parsePerSheet(value: string | undefined, separate?: boolean): number | undefined { @@ -254,14 +248,11 @@ function parsePreviewWindow(opts: MediaPreviewOptions): { start?: number; end?: async function mediaFilmstrip(ref: string, opts: MediaPreviewOptions): Promise { const { start, end, scale } = parsePreviewWindow(opts); const target = resolveAssetRef(ref); - const path = opts.output ?? join(tmpdir(), `${randomUUID()}.png`); - mkdirSync(dirname(resolve(path)), { recursive: true }); const stop = startSpinner("Rendering filmstrip"); try { - const { png, ...rest } = await call("media_filmstrip", { ...target, start, end, scale }); + const result = await call("media_filmstrip", { ...target, start, end, scale, output: resolveOutput(opts.output) }); stop(); - writeFileSync(path, png); - console.log(JSON.stringify({ path, ...rest })); + console.log(JSON.stringify(result)); } catch (e) { stop(); handleSocketError(e); @@ -271,14 +262,11 @@ async function mediaFilmstrip(ref: string, opts: MediaPreviewOptions): Promise { const { start, end, scale } = parsePreviewWindow(opts); const target = resolveAssetRef(ref); - const path = opts.output ?? join(tmpdir(), `${randomUUID()}.png`); - mkdirSync(dirname(resolve(path)), { recursive: true }); const stop = startSpinner("Rendering waveform"); try { - const { png, ...rest } = await call("media_waveform", { ...target, start, end, scale }); + const result = await call("media_waveform", { ...target, start, end, scale, output: resolveOutput(opts.output) }); stop(); - writeFileSync(path, png); - console.log(JSON.stringify({ path, ...rest })); + console.log(JSON.stringify(result)); } catch (e) { stop(); handleSocketError(e); @@ -292,11 +280,13 @@ async function captureNode(id: string, opts: CaptureOptions): Promise { const frames = times.map((t) => Math.round(t * TIME_FPS)); const perSheet = parsePerSheet(opts.perSheet, opts.separate); - const dir = opts.output ?? join(tmpdir(), `dapi-capture-${randomUUID().slice(0, 8)}`); - mkdirSync(dir, { recursive: true }); try { - const images = await call("capture", { id, frames, combine: !opts.separate, perSheet }, GENERATE); - writeImages(images, dir); + const { images } = await call( + "capture", + { id, frames, combine: !opts.separate, perSheet, output: resolveOutput(opts.output) }, + GENERATE, + ); + printImages(images); } catch (e) { handleSocketError(e); } @@ -346,7 +336,7 @@ async function openProject(path: string | undefined, opts: OpenOptions): Promise try { // A cold launch needs the renderer up before the app can answer; when // nothing was launched there is nothing to wait for, so fail fast. - if (launched) await waitForCliSocket(); + if (launched) await waitForApp(); else await ping(); if (path !== undefined) { @@ -369,8 +359,8 @@ async function context(): Promise { async function whoami(): Promise { try { - const result = await call("whoami", {}); - console.log(JSON.stringify(result)); + const { user } = await call("whoami", {}); + console.log(JSON.stringify(user)); } catch (e) { handleSocketError(e); } @@ -396,7 +386,7 @@ async function showLogs(opts: LogsOptions): Promise { } try { - const entries = await call("logs", { tail, level: opts.level as LogLevel | undefined }); + const { entries } = await call("logs", { tail, level: opts.level as LogLevel | undefined }); for (const entry of entries) console.log(formatLogEntry(entry)); } catch (e) { handleSocketError(e); @@ -414,27 +404,10 @@ function formatLogEntry(entry: LogEntry): string { type ScreenshotOptions = { output?: string }; // `diffusion-studio_2026-07-31_08-55-12.png` -function screenshotFilename(taken: Date, attempt: number): string { - const pad = (value: number) => String(value).padStart(2, "0"); - const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-"); - const time = [pad(taken.getHours()), pad(taken.getMinutes()), pad(taken.getSeconds())].join("-"); - const slug = APP_NAME.toLowerCase().replace(/[^a-z0-9]+/g, "-"); - return `${slug}_${date}_${time}${attempt > 1 ? `-${attempt}` : ""}.png`; -} - async function appScreenshot(opts: ScreenshotOptions): Promise { - const dir = opts.output ?? tmpdir(); - mkdirSync(dir, { recursive: true }); try { - const { png, width, height } = await call("screenshot", {}); - const taken = new Date(); - let attempt = 1; - let path = join(dir, screenshotFilename(taken, attempt)); - while (existsSync(path)) { - path = join(dir, screenshotFilename(taken, ++attempt)); - } - writeFileSync(path, png); - console.log(JSON.stringify({ path, width, height })); + const result = await call("screenshot", { output: resolveOutput(opts.output) }); + console.log(JSON.stringify(result)); } catch (e) { handleSocketError(e); } @@ -443,56 +416,21 @@ async function appScreenshot(opts: ScreenshotOptions): Promise { type IssueOptions = { body?: string; command?: string[]; logs?: string }; async function reportIssue(title: string, opts: IssueOptions): Promise { - const summary = title.trim(); - if (!summary) { - console.error("A one-line title is required."); - process.exit(1); - } - - let tail = ISSUE_LOG_TAIL; + let logs: number | undefined; if (opts.logs !== undefined) { const n = Number(opts.logs); if (!Number.isInteger(n) || n < 0) { console.error(`--logs must be a non-negative integer (got "${opts.logs}")`); process.exit(1); } - tail = n; + logs = n; } - - // The app being broken (or down) is exactly what gets reported, so a failed - // log read is recorded in the report rather than failing the command. - let logs: string[] | undefined; - let appStatus = "not checked"; - if (tail > 0) { - try { - logs = (await call("logs", { tail })).map(formatLogEntry); - appStatus = "running"; - } catch (e) { - const code = errnoCode(e); - appStatus = code === "ENOENT" || code === "ECONNREFUSED" - ? "not running" - : `unreachable (${(e as Error).message})`; - } - } - - const body = buildIssueBody({ - title: summary, - body: opts.body, - commands: opts.command, - logs, - appStatus, - version, - }); - - let url: string; try { - url = await createIssue(summary, body); + const result = await call("report", { title, body: opts.body, commands: opts.command, logs }); + console.log(JSON.stringify(result)); } catch (e) { - console.error((e as Error).message); - process.exit(1); + handleSocketError(e); } - - console.log(JSON.stringify({ url })); } function startSpinner(label: string): () => void { @@ -522,7 +460,7 @@ async function listModels(type: string | undefined): Promise { process.exit(1); } try { - const models = await call("models", { type: type as "image" | "video" | "audio" | undefined }); + const { models } = await call("models", { type: type as "image" | "video" | "audio" | undefined }); for (const model of models) console.log(JSON.stringify(model)); } catch (e) { handleSocketError(e); @@ -531,7 +469,7 @@ async function listModels(type: string | undefined): Promise { async function listVoices(): Promise { try { - const voices = await call("voices", {}); + const { voices } = await call("voices", {}); for (const voice of voices) console.log(JSON.stringify(voice)); } catch (e) { handleSocketError(e); @@ -546,7 +484,7 @@ type ListFontsOptions = { namesOnly?: boolean; }; -function listFonts(opts: ListFontsOptions): void { +async function listFonts(opts: ListFontsOptions): Promise { let style: "normal" | "italic" | undefined; if (opts.style !== undefined) { if (opts.style !== "normal" && opts.style !== "italic") { @@ -567,34 +505,29 @@ function listFonts(opts: ListFontsOptions): void { } try { - const families = listLocalFonts({ - familyPattern: opts.family, - weights: opts.weight, - style, - limit, - }); + const { families } = await call("fonts", { family: opts.family, weights: opts.weight, style, limit }); if (opts.namesOnly) { for (const family of families) console.log(family.family); } else { for (const family of families) console.log(JSON.stringify(family)); } } catch (e) { - console.error((e as Error).message); - process.exit(1); + handleSocketError(e); } } type FetchCliOptions = { output?: string; format?: string; audio?: boolean }; // `raw` is every operand after `url` — the yt-dlp passthrough placed after `--`. -// No spinner here: yt-dlp renders its own progress to the inherited stderr. async function fetch(url: string, opts: FetchCliOptions, raw: string[]): Promise { + const stop = startSpinner("Downloading"); try { - const paths = await fetchVideo(url, { ...opts, raw }); + const { paths } = await call("fetch", { url, ...opts, raw, output: resolveOutput(opts.output) }, GENERATE); + stop(); for (const path of paths) console.log(JSON.stringify({ path })); } catch (e) { - console.error((e as Error).message); - process.exit(1); + stop(); + handleSocketError(e); } } @@ -788,7 +721,7 @@ program program .command("fonts") .description( - `List the local fonts available on this machine (macOS only; does not require the app). These family names are valid \`fontFamily\` values on ; each family lists its variants.`, + `List the local fonts available on this machine (macOS only). These family names are valid \`fontFamily\` values on ; each family lists its variants.`, ) .option("-f, --family ", "filter to families whose name contains (case-insensitive)") .option("-w, --weight ", "filter to variants with the given CSS weight(s), e.g. -w 400 700") @@ -800,7 +733,7 @@ program program .command("fetch") .description( - `Download a video with yt-dlp (installed separately; does not require the app). Writes files to disk only (a single URL can yield several, e.g. a playlist).`, + `Download a video with yt-dlp (installed separately). Writes files to disk only (a single URL can yield several, e.g. a playlist).`, ) .argument("", "video or page URL to download") .option("-o, --output ", "output file path or directory (yt-dlp -o template; default: yt-dlp's default)") diff --git a/apps/cli/src/protocol.ts b/apps/cli/src/protocol.ts deleted file mode 100644 index eda36e18..00000000 --- a/apps/cli/src/protocol.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -// The transport between the CLI and the app, and nothing else: the tool -// catalog and every request and result type live in @diffusionstudio/dapi. -// Free of Node built-ins so the renderer can import it; the socket path is -// in @diffusionstudio/dapi/socket. -// -// Each CLI command hosts a short-lived WebSocket server; main's only job is -// to relay the connect info to the renderer, which then dials the CLI -// directly. Main never sees request payloads. -export const CLI_WIRE = { - CONNECT: "cli:connect", -} as const; - -// Sent by the CLI to main over the unix socket, relayed verbatim to the -// renderer. The token guards the loopback WebSocket server against other -// local processes racing to connect first. -export type CliHandshake = { port: number; token: string }; - -export type CliHandshakeReply = { ok: true } | { ok: false; error: string }; - -// One tRPC request/reply pair per WebSocket connection. `path` is the -// dot-joined procedure path in the renderer's router (e.g. "media.frame"); -// procedure inputs and outputs are typed end-to-end via the AppRouter type, -// so the wire envelope stays untyped. -export type CliRequest = { - path: string; - input: unknown; -}; - -export type CliReply = - | { ok: true; data: unknown } - | { ok: false; error: string }; - -// Replies carry bytes (PNGs) inside JSON. A Uint8Array is written as -// `{ $bytes: }` and read back as a Uint8Array, so handlers and the -// CLI both see bytes and only this seam knows about base64. Chunked so a -// 100 MiB frame batch never builds a single giant argument list. -const BYTES_KEY = "$bytes"; -const CHUNK = 0x8000; - -export function encodeReply(reply: CliReply): string { - return JSON.stringify(reply, (_key, value) => - value instanceof Uint8Array ? { [BYTES_KEY]: bytesToBase64(value) } : value, - ); -} - -export function decodeReply(text: string): CliReply { - return JSON.parse(text, (_key, value) => - isBytesEnvelope(value) ? base64ToBytes(value[BYTES_KEY]) : value, - ) as CliReply; -} - -function isBytesEnvelope(value: unknown): value is { [BYTES_KEY]: string } { - return typeof value === "object" && value !== null && typeof (value as Record)[BYTES_KEY] === "string"; -} - -function bytesToBase64(bytes: Uint8Array): string { - let binary = ""; - for (let i = 0; i < bytes.length; i += CHUNK) { - binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)); - } - return btoa(binary); -} - -function base64ToBytes(base64: string): Uint8Array { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return bytes; -} diff --git a/apps/cli/src/ytdlp.ts b/apps/cli/src/ytdlp.ts deleted file mode 100644 index c184a2b4..00000000 --- a/apps/cli/src/ytdlp.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { spawn, spawnSync } from "node:child_process"; - -// Resolve the binary once. YT_DLP_PATH mirrors DIFFUSION_APP_PATH: an escape -// hatch for pinned or non-PATH installs. -const BIN = process.env.YT_DLP_PATH ?? "yt-dlp"; - -// Fails with an actionable message before any download is attempted. ENOENT is -// the "not installed" case; a non-zero status means the binary is present but -// broken. -function assertInstalled(): void { - const probe = spawnSync(BIN, ["--version"], { encoding: "utf8" }); - if (probe.error) { - if ((probe.error as NodeJS.ErrnoException).code === "ENOENT") { - throw new Error( - `yt-dlp not found. Install it (brew install yt-dlp, or pipx install yt-dlp) ` + - `or set YT_DLP_PATH to its location.`, - ); - } - throw new Error(probe.error.message); - } - if (probe.status !== 0) { - throw new Error(probe.stderr?.trim() || "yt-dlp is present but not runnable."); - } -} - -export type FetchOptions = { - output?: string; - format?: string; - audio?: boolean; - raw?: string[]; -}; - -// Downloads the URL and resolves to the final file path(s) on disk. yt-dlp's -// own `after_move:filepath` reports what actually landed (post extraction / -// rename), so we never guess the name. --quiet silences stdout chatter while -// --progress forces the progress bar to stderr, keeping stdout to just the -// paths we capture. -export function fetchVideo(url: string, opts: FetchOptions = {}): Promise { - assertInstalled(); - - const args = ["--quiet", "--no-warnings", "--progress", "--print", "after_move:filepath"]; - if (opts.output) args.push("-o", opts.output); - if (opts.format) { - args.push("-f", opts.format); - } else if (!opts.audio) { - // Default to mp4: prefer mp4/m4a streams, then remux the merged result so - // the landed file is a .mp4 even when only WebM/mkv sources were available. - // An explicit -f or --audio opts out. - args.push("-f", "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/bv*+ba/b", "--merge-output-format", "mp4"); - } - if (opts.audio) args.push("-x"); - if (opts.raw?.length) args.push(...opts.raw); - args.push(url); - - return new Promise((resolve, reject) => { - // stdout: resolved paths (captured); stderr: live progress (inherited). - const child = spawn(BIN, args, { stdio: ["ignore", "pipe", "inherit"] }); - let out = ""; - child.stdout.setEncoding("utf8"); - child.stdout.on("data", (chunk) => (out += chunk)); - child.on("error", reject); - child.on("close", (code) => { - if (code === 0) { - resolve(out.split("\n").map((line) => line.trim()).filter(Boolean)); - } else { - reject(new Error(`yt-dlp exited with code ${code}.`)); - } - }); - }); -} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 56a49222..09d4b1e6 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -22,7 +22,8 @@ "package": "npm run build && npm run build:web && npm run stage:cli && npm run stage:docs && npm run stage:skills && electron-forge package", "make": "npm run build && npm run build:web && npm run stage:cli && npm run stage:docs && npm run stage:skills && electron-forge make", "publish": "npm run build && npm run build:web && npm run stage:cli && npm run stage:docs && npm run stage:skills && electron-forge publish", - "make:icns": "sh scripts/make-icns.sh" + "make:icns": "sh scripts/make-icns.sh", + "test": "vitest run" }, "devDependencies": { "@electron-forge/cli": "^7.11.1", @@ -32,19 +33,20 @@ "@types/babel__core": "^7.20.5", "@types/node": "^24.10.1", "electron": "^43.1.1", - "typescript": "~5.9.3" + "typescript": "~5.9.3", + "vitest": "^4.0.0" }, "dependencies": { "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", - "@diffusionstudio/cli": "*", + "@diffusionstudio/dapi": "*", "@diffusionstudio/jsx": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "babel-preset-solid": "^1.9.12", "esbuild": "^0.28.1", "nanoid": "^6.0.1", "ts-morph": "^28.0.0", "update-electron-app": "^3.0.0", - "yaml": "^2.9.0", - "@diffusionstudio/dapi": "*" + "yaml": "^2.9.0" } } diff --git a/apps/desktop/scripts/stage-cli.mjs b/apps/desktop/scripts/stage-cli.mjs index 90590cda..2183f24f 100644 --- a/apps/desktop/scripts/stage-cli.mjs +++ b/apps/desktop/scripts/stage-cli.mjs @@ -17,10 +17,7 @@ const desktopDir = join(dirname(fileURLToPath(import.meta.url)), ".."); const cliDir = join(desktopDir, "..", "cli"); const stageDir = join(desktopDir, "cli"); -// Must match the --external list in the apps/cli build script, minus ws's -// optional native addons (bufferutil, utf-8-validate): those are external -// only so the bundle doesn't choke on them, and ws falls back to its JS -// implementations when they are absent. +// Must match the --external list in the apps/cli build script. const EXTERNALS = ["esbuild", "@babel/core", "@babel/preset-typescript", "babel-preset-solid"]; const cliPkg = JSON.parse(readFileSync(join(cliDir, "package.json"), "utf8")); diff --git a/apps/desktop/src/cli-server.ts b/apps/desktop/src/cli-server.ts deleted file mode 100644 index 457cb750..00000000 --- a/apps/desktop/src/cli-server.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { existsSync, unlinkSync } from "node:fs"; -import { createServer } from "node:net"; -import type { Server, Socket } from "node:net"; -import { app, BrowserWindow } from "electron"; -import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; -import { SOCKET_PATH } from "@diffusionstudio/dapi/socket"; -import type { CliHandshake, CliHandshakeReply } from "@diffusionstudio/cli/protocol"; -import { mainBridge } from "./main-manager"; -import { MAIN_CHANNELS } from "./main-channels"; - -let cliServer: Server | null = null; -let currentWindow: BrowserWindow | null = null; -let headless = false; -let windowLifecycleBound = false; - -export function isHeadless(): boolean { - return headless; -} - -function enableHeadless(): void { - if (headless) return; - headless = true; - if (currentWindow && !currentWindow.isDestroyed()) { - mainBridge.emit(currentWindow, MAIN_CHANNELS.HEADLESS_MODE, { active: true }); - } -} - -// Resolves once the current window has finished loading. -function waitForRendererReady(timeoutMs = 30000): Promise { - if (!currentWindow || currentWindow.isDestroyed()) { - return Promise.reject(new Error("No window")); - } - - if (currentWindow.webContents.isCrashed()) { - return Promise.reject(new Error("Renderer crashed")); - } - if (!currentWindow.webContents.isLoading()) { - return Promise.resolve(); - } - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - console.warn(`[cli-server] renderer not ready after ${timeoutMs}ms`); - reject(new Error("App did not become ready in time")); - }, timeoutMs); - - const cleanup = () => { - clearTimeout(timer); - currentWindow?.webContents.off("did-finish-load", onLoad); - currentWindow?.webContents.off("did-fail-load", onFail); - }; - - const onLoad = () => { - cleanup(); - resolve(); - }; - - const onFail = () => { - cleanup(); - reject(new Error(`Renderer failed to load`)); - }; - - currentWindow?.webContents.on("did-finish-load", onLoad); - currentWindow?.webContents.on("did-fail-load", onFail); - }); -} - -function bindWindowLifecycle(): void { - if (windowLifecycleBound) return; - windowLifecycleBound = true; - app.on("browser-window-created", (_event, window) => { - currentWindow = window; - window.on("closed", () => { - if (currentWindow === window) currentWindow = null; - }); - }); - // Catch any window that was created before the server started. - const windows = BrowserWindow.getAllWindows(); - if (windows.length > 0) currentWindow = windows[windows.length - 1]!; -} - -// Relays the CLI's connect info to the renderer, which dials the CLI's -// WebSocket server directly. The reply confirms delivery only; from there -// the request/response traffic bypasses main entirely. -async function deliverHandshake(handshake: CliHandshake, sock: Socket): Promise { - let reply: CliHandshakeReply; - try { - await waitForRendererReady(); - if (!currentWindow || currentWindow.isDestroyed()) { - throw new Error("No window"); - } - currentWindow.webContents.send(CLI_WIRE.CONNECT, handshake); - reply = { ok: true }; - } catch (err) { - reply = { ok: false, error: (err as Error).message }; - } - if (!sock.destroyed) sock.end(JSON.stringify(reply)); -} - -export function startCliServer() { - cleanupStaleSocket(); - bindWindowLifecycle(); - - cliServer = createServer({ allowHalfOpen: true }, (sock: Socket) => { - enableHeadless(); - let buf = ""; - sock.setEncoding("utf8"); - sock.setTimeout(60000, () => sock.destroy()); - sock.on("data", (chunk) => { - buf += chunk; - }); - sock.on("end", async () => { - sock.setTimeout(0); - let handshake: CliHandshake; - try { - handshake = JSON.parse(buf) as CliHandshake; - if (typeof handshake.port !== "number" || typeof handshake.token !== "string") { - throw new Error("Malformed handshake"); - } - } catch { - sock.end(JSON.stringify({ ok: false, error: "Invalid handshake" })); - return; - } - await deliverHandshake(handshake, sock); - }); - sock.on("error", () => { - // Client hung up; nothing to do. - }); - }); - - cliServer.on("error", (err) => { - console.error("CLI server error:", err); - }); - - cliServer.listen(SOCKET_PATH); -} - -export function stopCliServer() { - if (!cliServer) return; - cliServer.close(); - cliServer = null; - cleanupStaleSocket(); -} - -/** - * Clean up a stale socket file (Unix). Safe because the single-instance lock - * guarantees no other instance of ours is running. - */ -function cleanupStaleSocket() { - try { - if (process.platform !== "win32" && existsSync(SOCKET_PATH)) { - unlinkSync(SOCKET_PATH); - } - } catch { - // Best-effort. - } -} diff --git a/apps/desktop/src/dapi/handler.ts b/apps/desktop/src/dapi/handler.ts new file mode 100644 index 00000000..f94d9c72 --- /dev/null +++ b/apps/desktop/src/dapi/handler.ts @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { LogEntry, ToolArgs, ToolResult } from "@diffusionstudio/dapi"; + +/** What a main-process handler gets besides its arguments. */ +export type MainContext = { + /** Fires when the caller cancels or goes away. */ + signal: AbortSignal; + /** The app's console buffer, oldest first. */ + logs(): LogEntry[]; + /** The app's version. */ + version: string; +}; + +/** The tools that need the file system or a child process, not a window. */ +export type MainToolName = "logs" | "fonts" | "fetch" | "report"; + +export type MainHandler = (args: ToolArgs, ctx: MainContext) => Promise>; + +export type MainHandlers = { readonly [N in MainToolName]: MainHandler }; diff --git a/apps/desktop/src/dapi/handlers/fetch.ts b/apps/desktop/src/dapi/handlers/fetch.ts new file mode 100644 index 00000000..a809bf23 --- /dev/null +++ b/apps/desktop/src/dapi/handlers/fetch.ts @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { spawn, spawnSync } from "node:child_process"; +import { DapiError } from "@diffusionstudio/dapi"; + +import type { MainHandler } from "../handler"; + +// Resolve the binary once. YT_DLP_PATH is an escape hatch for pinned or +// non-PATH installs. +const BIN = process.env.YT_DLP_PATH ?? "yt-dlp"; + +// Fails with an actionable message before any download is attempted. ENOENT is +// the "not installed" case; a non-zero status means the binary is present but +// broken. +function assertInstalled(): void { + const probe = spawnSync(BIN, ["--version"], { encoding: "utf8" }); + if (probe.error) { + if ((probe.error as NodeJS.ErrnoException).code === "ENOENT") { + throw new DapiError( + "unsupported", + "yt-dlp not found. Install it (brew install yt-dlp, or pipx install yt-dlp) or set YT_DLP_PATH to its location.", + ); + } + throw probe.error; + } + if (probe.status !== 0) { + throw new DapiError("unsupported", probe.stderr?.trim() || "yt-dlp is present but not runnable."); + } +} + +// yt-dlp's own `after_move:filepath` reports what actually landed (post +// extraction / rename), so the name is never guessed. --quiet keeps stdout to +// those paths; stderr is kept only for the error line when it fails. +export const fetchVideo: MainHandler<"fetch"> = ({ url, output, format, audio, raw }, ctx) => { + assertInstalled(); + + const args = ["--quiet", "--no-warnings", "--print", "after_move:filepath"]; + if (output) args.push("-o", output); + if (format) { + args.push("-f", format); + } else if (!audio) { + // Default to mp4: prefer mp4/m4a streams, then remux the merged result so + // the landed file is a .mp4 even when only WebM/mkv sources were available. + // An explicit format or audio opts out. + args.push("-f", "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/bv*+ba/b", "--merge-output-format", "mp4"); + } + if (audio) args.push("-x"); + if (raw?.length) args.push(...raw); + args.push(url); + + return new Promise((resolve, reject) => { + const child = spawn(BIN, args, { stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk) => (out += chunk)); + + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + stderr = (stderr + chunk).slice(-4096); + }); + + const onAbort = () => child.kill(); + ctx.signal.addEventListener("abort", onAbort, { once: true }); + + child.on("error", reject); + child.on("close", (code) => { + ctx.signal.removeEventListener("abort", onAbort); + if (ctx.signal.aborted) { + reject(new DapiError("canceled", "Download canceled.")); + } else if (code === 0) { + resolve({ paths: out.split("\n").map((line) => line.trim()).filter(Boolean) }); + } else { + const detail = stderr.trim().split("\n").filter((l) => l.startsWith("ERROR")).pop(); + reject(new Error(detail ?? `yt-dlp exited with code ${code}.`)); + } + }); + }); +}; diff --git a/apps/cli/src/fonts.ts b/apps/desktop/src/dapi/handlers/fonts.ts similarity index 66% rename from apps/cli/src/fonts.ts rename to apps/desktop/src/dapi/handlers/fonts.ts index d2de7964..56f54b5f 100644 --- a/apps/cli/src/fonts.ts +++ b/apps/desktop/src/dapi/handlers/fonts.ts @@ -4,21 +4,14 @@ import { spawnSync } from "node:child_process"; import { platform } from "node:os"; +import { DapiError } from "@diffusionstudio/dapi"; -export type FontVariant = { - weight: string; - style: "normal" | "italic"; - source: string; -}; - -export type FontFamily = { - family: string; - variants: FontVariant[]; -}; +import type { FontFamily } from "@diffusionstudio/dapi"; +import type { MainHandler } from "../handler"; // JXA script that walks every registered font family via NSFontManager and // emits each variant's CSS-style weight, italic flag, and CSS `local()` source. -// Embedded inline so the CLI binary is self-contained — runs via `osascript`. +// Runs via `osascript`. const LIST_FONTS_JXA = ` ObjC.import("AppKit"); @@ -63,16 +56,9 @@ function run() { } `; -export type ListLocalFontsOptions = { - familyPattern?: string; - weights?: string[]; - style?: "normal" | "italic"; - limit?: number; -}; - -export function listLocalFonts(options: ListLocalFontsOptions = {}): FontFamily[] { +function listLocalFonts(): FontFamily[] { if (platform() !== "darwin") { - throw new Error("fonts is only supported on macOS."); + throw new DapiError("unsupported", "fonts is only supported on macOS."); } const result = spawnSync("osascript", ["-l", "JavaScript", "-e", LIST_FONTS_JXA], { encoding: "utf8", @@ -81,23 +67,24 @@ export function listLocalFonts(options: ListLocalFontsOptions = {}): FontFamily[ if (result.status !== 0) { throw new Error(result.stderr.trim() || "Failed to enumerate fonts."); } + return JSON.parse(result.stdout.trim()) as FontFamily[]; +} - const all = JSON.parse(result.stdout.trim()) as FontFamily[]; - const pattern = options.familyPattern?.toLowerCase(); - const weights = options.weights && options.weights.length > 0 ? new Set(options.weights) : null; - const { style, limit } = options; +export const fonts: MainHandler<"fonts"> = async ({ family, weights, style, limit }) => { + const pattern = family?.toLowerCase(); + const wanted = weights && weights.length > 0 ? new Set(weights) : null; - const out: FontFamily[] = []; - for (const family of all) { - if (pattern && !family.family.toLowerCase().includes(pattern)) continue; - const variants = family.variants.filter((v) => { - if (weights && !weights.has(v.weight)) return false; + const families: FontFamily[] = []; + for (const entry of listLocalFonts()) { + if (pattern && !entry.family.toLowerCase().includes(pattern)) continue; + const variants = entry.variants.filter((v) => { + if (wanted && !wanted.has(v.weight)) return false; if (style && v.style !== style) return false; return true; }); if (variants.length === 0) continue; - out.push({ family: family.family, variants }); - if (limit !== undefined && out.length >= limit) break; + families.push({ family: entry.family, variants }); + if (limit !== undefined && families.length >= limit) break; } - return out; -} + return { families }; +}; diff --git a/apps/desktop/src/dapi/handlers/index.ts b/apps/desktop/src/dapi/handlers/index.ts new file mode 100644 index 00000000..0faa91ad --- /dev/null +++ b/apps/desktop/src/dapi/handlers/index.ts @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { logs } from "./logs"; +import { fonts } from "./fonts"; +import { fetchVideo } from "./fetch"; +import { report } from "./report"; + +import type { MainHandlers } from "../handler"; + +/** Every tool main answers itself, keyed by its catalog name. */ +export const mainHandlers: MainHandlers = { logs, fonts, fetch: fetchVideo, report }; diff --git a/apps/desktop/src/dapi/handlers/logs.ts b/apps/desktop/src/dapi/handlers/logs.ts new file mode 100644 index 00000000..7b04e6af --- /dev/null +++ b/apps/desktop/src/dapi/handlers/logs.ts @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { LogEntry, LogLevel } from "@diffusionstudio/dapi"; +import type { MainHandler } from "../handler"; + +const LEVEL_RANK: Record = { debug: 0, info: 1, warning: 2, error: 3 }; + +export const logs: MainHandler<"logs"> = async ({ tail, level }, ctx) => { + let entries = ctx.logs(); + if (level !== undefined) { + const min = LEVEL_RANK[level]; + entries = entries.filter((e) => LEVEL_RANK[e.level] >= min); + } + if (tail !== undefined) entries = entries.slice(-tail); + return { entries }; +}; + +/** One log entry as a line: local time, level, message, source. */ +export function formatLogEntry(entry: LogEntry): string { + const pad = (n: number, w = 2) => String(n).padStart(w, "0"); + const d = new Date(entry.ts); + const time = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}.${pad(d.getMilliseconds(), 3)}`; + const source = entry.source ? ` (${entry.source})` : ""; + return `${time} [${entry.level}] ${entry.message}${source}`; +} diff --git a/apps/cli/src/report.ts b/apps/desktop/src/dapi/handlers/report.ts similarity index 58% rename from apps/cli/src/report.ts rename to apps/desktop/src/dapi/handlers/report.ts index d618dec5..15876e06 100644 --- a/apps/cli/src/report.ts +++ b/apps/desktop/src/dapi/handlers/report.ts @@ -4,50 +4,41 @@ import { spawn } from "node:child_process"; import { arch, platform, release } from "node:os"; +import { DapiError, ISSUE_LOG_TAIL } from "@diffusionstudio/dapi"; +import { formatLogEntry } from "./logs"; + +import type { MainHandler } from "../handler"; const REPO = "diffusionstudio/editor"; -export const GH_MISSING = +const GH_MISSING = "gh (GitHub CLI) is not installed, so the issue cannot be filed. Install it from https://cli.github.com, run `gh auth login`, then retry."; -export type IssueInput = { - title: string; - body?: string; - commands?: string[]; - logs?: string[]; // already formatted log lines, oldest first - appStatus: string; // "running", "not running", "not checked", or why it was unreachable - version: string; -}; - function fence(language: string, content: string): string { return `\`\`\`${language}\n${content}\n\`\`\``; } -function environmentTable(input: IssueInput): string { +function environmentTable(version: string): string { const rows: Array<[string, string]> = [ - ["dapi", input.version], + ["app", version], ["platform", `${platform()} ${release()} (${arch()})`], - ["node", process.version], - ["app", input.appStatus], + ["electron", process.versions.electron ?? "unknown"], ]; return ["| | |", "| --- | --- |", ...rows.map(([k, v]) => `| ${k} | ${v} |`)].join("\n"); } -export function buildIssueBody(input: IssueInput): string { +function buildIssueBody(input: { body?: string; commands?: string[]; logs: string[]; version: string }): string { const sections: string[] = []; - if (input.body?.trim()) sections.push(input.body.trim()); if (input.commands?.length) sections.push(`## Repro\n\n${fence("sh", input.commands.join("\n"))}`); - sections.push(`## Environment\n\n${environmentTable(input)}`); - if (input.logs?.length) sections.push(`## App logs\n\n${fence("", input.logs.join("\n"))}`); - + sections.push(`## Environment\n\n${environmentTable(input.version)}`); + if (input.logs.length) sections.push(`## App logs\n\n${fence("", input.logs.join("\n"))}`); return `${sections.join("\n\n")}\n`; } -// --repo is explicit because dapi runs from the user's project, not a checkout -// of the editor; the body goes over stdin so a long log tail can't blow the -// argv size limit. -export function createIssue(title: string, body: string): Promise { +// --repo is explicit because the app is not a checkout of the editor; the body +// goes over stdin so a long log tail can't blow the argv size limit. +function createIssue(title: string, body: string): Promise { return new Promise((resolve, reject) => { const gh = spawn("gh", ["issue", "create", "--repo", REPO, "--title", title, "--body-file", "-"]); @@ -57,7 +48,7 @@ export function createIssue(title: string, body: string): Promise { gh.stderr.on("data", (chunk) => (stderr += chunk)); gh.on("error", (e) => { - reject((e as NodeJS.ErrnoException).code === "ENOENT" ? new Error(GH_MISSING) : e); + reject((e as NodeJS.ErrnoException).code === "ENOENT" ? new DapiError("unsupported", GH_MISSING) : e); }); gh.on("close", (code) => { if (code !== 0) { @@ -73,7 +64,16 @@ export function createIssue(title: string, body: string): Promise { resolve(url); }); - gh.stdin.on("error", () => { }); // gh exiting early (auth failure) breaks the pipe + gh.stdin.on("error", () => {}); // gh exiting early (auth failure) breaks the pipe gh.stdin.end(body); }); } + +export const report: MainHandler<"report"> = async ({ title, body, commands, logs }, ctx) => { + const summary = title.trim(); + if (!summary) throw new DapiError("invalid-input", "A one-line title is required."); + const tail = logs ?? ISSUE_LOG_TAIL; + const lines = tail > 0 ? ctx.logs().slice(-tail).map(formatLogEntry) : []; + const url = await createIssue(summary, buildIssueBody({ body, commands, logs: lines, version: ctx.version })); + return { url }; +}; diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.ts new file mode 100644 index 00000000..1c75de82 --- /dev/null +++ b/apps/desktop/src/dapi/present.test.ts @@ -0,0 +1,68 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { present, toCallToolResult } from "./present"; + +const dir = mkdtempSync(join(tmpdir(), "dapi-present-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +const png = (byte: number, size = 8) => new Uint8Array(size).fill(byte); + +describe("present", () => { + it("writes capture frames by timecode into the requested directory and returns their paths", async () => { + const out = join(dir, "frames"); + const presented = await present("capture", { id: "intro", combine: true, output: out }, [ + { timecode: "0f", png: png(1) }, + { timecode: "1s", png: png(2) }, + ]); + expect(presented.output).toEqual({ images: [{ timecode: "0f", path: join(out, "0f.png") }, { timecode: "1s", path: join(out, "1s.png") }] }); + expect(readFileSync(join(out, "1s.png"))).toEqual(Buffer.from(png(2))); + }); + + it("picks a fresh temp directory when none is given", async () => { + const presented = await present("media_grab", { path: "/c.mp4", combine: true }, [{ timecode: "0f", png: png(3) }]); + const { path } = (presented.output as { images: Array<{ path: string }> }).images[0]!; + expect(path).toMatch(/dapi-grab-.*[\\/]0f\.png$/); + rmSync(join(path, ".."), { recursive: true, force: true }); + }); + + it("keeps a preview's other fields next to the path", async () => { + const file = join(dir, "wave.png"); + const presented = await present("media_waveform", { path: "/c.mp4", output: file }, { png: png(4), silences: [{ start: 0, end: 1 }] }); + expect(presented.output).toEqual({ path: file, silences: [{ start: 0, end: 1 }] }); + }); + + it("names screenshots by time and never overwrites one", async () => { + const first = await present("screenshot", { output: dir }, { png: png(5), width: 10, height: 10 }); + const second = await present("screenshot", { output: dir }, { png: png(6), width: 10, height: 10 }); + const a = (first.output as { path: string }).path; + const b = (second.output as { path: string }).path; + expect(a).toMatch(/diffusion-studio_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.png$/); + expect(b).not.toBe(a); + }); + + it("passes other results through untouched", async () => { + expect(await present("check", { id: "x" }, { stats: {}, issues: [] })).toEqual({ output: { stats: {}, issues: [] }, images: [] }); + }); +}); + +describe("toCallToolResult", () => { + it("inlines a few small images and always carries the output as text and structure", () => { + const result = toCallToolResult({ output: { images: [] }, images: [{ path: "/a.png", png: png(1) }] }); + expect(result.structuredContent).toEqual({ images: [] }); + expect(result.content[0]).toEqual({ type: "text", text: '{"images":[]}' }); + expect(result.content[1]).toMatchObject({ type: "image", mimeType: "image/png" }); + }); + + it("sends paths only when there are many images or a large one", () => { + const many = Array.from({ length: 5 }, (_, i) => ({ path: `/${i}.png`, png: png(i) })); + expect(toCallToolResult({ output: {}, images: many }).content).toHaveLength(1); + const large = [{ path: "/big.png", png: png(0, (1 << 20) + 1) }]; + expect(toCallToolResult({ output: {}, images: large }).content).toHaveLength(1); + }); +}); diff --git a/apps/desktop/src/dapi/present.ts b/apps/desktop/src/dapi/present.ts new file mode 100644 index 00000000..67d90d1f --- /dev/null +++ b/apps/desktop/src/dapi/present.ts @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Turns what a handler returned into what a caller receives. Tools that +// render images hand back bytes; here they become files on disk, paths in +// the structured result, and — when the result is small — inline images the +// agent sees without opening anything. + +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { randomUUID } from "node:crypto"; + +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import type { TimecodedImage, ToolArgs, ToolName, ToolOutput, ToolResult } from "@diffusionstudio/dapi"; + +/** A file the tool wrote, kept in memory only long enough to decide whether to inline it. */ +export type WrittenImage = { path: string; png: Uint8Array }; + +export type Presented = { output: unknown; images: WrittenImage[] }; + +/** More than this, or any image larger than INLINE_MAX_BYTES, and the caller gets paths only. */ +const INLINE_MAX_IMAGES = 4; +const INLINE_MAX_BYTES = 1 << 20; + +const APP_SLUG = "diffusion-studio"; + +export async function present(name: ToolName, args: unknown, result: unknown): Promise { + switch (name) { + case "capture": + return presentImages(result as ToolResult<"capture">, (args as ToolArgs<"capture">).output, "capture"); + case "media_grab": + return presentImages(result as ToolResult<"media_grab">, (args as ToolArgs<"media_grab">).output, "grab"); + case "media_filmstrip": + return presentPreview(result as ToolResult<"media_filmstrip">, (args as ToolArgs<"media_filmstrip">).output, "filmstrip"); + case "media_waveform": + return presentPreview(result as ToolResult<"media_waveform">, (args as ToolArgs<"media_waveform">).output, "waveform"); + case "screenshot": + return presentScreenshot(result as ToolResult<"screenshot">, (args as ToolArgs<"screenshot">).output); + default: + return { output: result, images: [] }; + } +} + +// Frames and contact sheets arrive in the same shape: each image is stamped +// with its timecode (`08s10f`, or `0f-08s10f` for a sheet), which is the +// filename too. +async function presentImages(images: TimecodedImage[], output: string | undefined, kind: string): Promise { + const dir = output ?? (await mkdtemp(join(tmpdir(), `dapi-${kind}-`))); + await mkdir(dir, { recursive: true }); + const written: WrittenImage[] = []; + const refs: ToolOutput<"capture">["images"] = []; + for (const { timecode, png } of images) { + const path = join(dir, `${timecode}.png`); + await writeFile(path, png); + written.push({ path, png }); + refs.push({ timecode, path }); + } + return { output: { images: refs }, images: written }; +} + +async function presentPreview( + result: { png: Uint8Array } & Record, + output: string | undefined, + kind: string, +): Promise { + const { png, ...rest } = result; + const path = output ?? join(tmpdir(), `dapi-${kind}-${randomUUID()}.png`); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, png); + return { output: { path, ...rest }, images: [{ path, png }] }; +} + +async function presentScreenshot(result: ToolResult<"screenshot">, output: string | undefined): Promise { + const dir = output ?? tmpdir(); + await mkdir(dir, { recursive: true }); + const taken = new Date(); + let attempt = 1; + let path = join(dir, screenshotFilename(taken, attempt)); + while (existsSync(path)) path = join(dir, screenshotFilename(taken, ++attempt)); + await writeFile(path, result.png); + const presented: ToolOutput<"screenshot"> = { path, width: result.width, height: result.height }; + return { output: presented, images: [{ path, png: result.png }] }; +} + +function screenshotFilename(taken: Date, attempt: number): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = [taken.getFullYear(), pad(taken.getMonth() + 1), pad(taken.getDate())].join("-"); + const time = [pad(taken.getHours()), pad(taken.getMinutes()), pad(taken.getSeconds())].join("-"); + return `${APP_SLUG}_${date}_${time}${attempt > 1 ? `-${attempt}` : ""}.png`; +} + +/** The MCP result: the output as text and structured content, plus the images when they are few and small. */ +export function toCallToolResult({ output, images }: Presented): CallToolResult { + const content: CallToolResult["content"] = [{ type: "text", text: JSON.stringify(output) }]; + const inline = images.length <= INLINE_MAX_IMAGES && images.every((image) => image.png.byteLength <= INLINE_MAX_BYTES); + if (inline) { + for (const { png } of images) { + content.push({ type: "image", data: Buffer.from(png).toString("base64"), mimeType: "image/png" }); + } + } + return { content, structuredContent: output as Record }; +} + +/** A failure the agent reads as a sentence, not a protocol error. */ +export function toErrorResult(error: unknown): CallToolResult { + return { isError: true, content: [{ type: "text", text: (error as Error).message }] }; +} diff --git a/apps/desktop/src/dapi/renderer-calls.ts b/apps/desktop/src/dapi/renderer-calls.ts new file mode 100644 index 00000000..e5be4f51 --- /dev/null +++ b/apps/desktop/src/dapi/renderer-calls.ts @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { randomUUID } from "node:crypto"; +import { app, BrowserWindow, ipcMain } from "electron"; +import { DAPI_WIRE, DapiError } from "@diffusionstudio/dapi"; + +import type { DapiCall, DapiCancel, DapiReply } from "@diffusionstudio/dapi"; + +type InFlight = { + resolve(data: unknown): void; + reject(error: Error): void; +}; + +/** + * Runs renderer tools from main: one `dapi:call` per request, answered by + * `dapi:reply`, or `dapi:cancel` if the caller gives up. Calls wait for the + * current window to finish loading; a window that reloads or dies fails the + * calls it was answering. + */ +export class RendererCalls { + private readonly inFlight = new Map(); + private window: BrowserWindow | null = null; + + start(): void { + ipcMain.on(DAPI_WIRE.REPLY, (_event, reply: DapiReply) => { + const call = this.inFlight.get(reply.id); + if (!call) return; + this.inFlight.delete(reply.id); + if (reply.ok) call.resolve(reply.data); + else call.reject(reply.error.code ? new DapiError(reply.error.code, reply.error.message) : new Error(reply.error.message)); + }); + + app.on("browser-window-created", (_event, window) => this.track(window)); + const windows = BrowserWindow.getAllWindows(); + if (windows.length > 0) this.track(windows[windows.length - 1]!); + } + + async call(tool: string, args: unknown, signal: AbortSignal): Promise { + const window = await this.ready(); + const id = randomUUID(); + return new Promise((resolve, reject) => { + const onAbort = () => { + this.inFlight.delete(id); + if (!window.isDestroyed()) window.webContents.send(DAPI_WIRE.CANCEL, { id } satisfies DapiCancel); + reject(new DapiError("canceled", "The call was canceled.")); + }; + signal.addEventListener("abort", onAbort, { once: true }); + this.inFlight.set(id, { + resolve: (data) => { + signal.removeEventListener("abort", onAbort); + resolve(data); + }, + reject: (error) => { + signal.removeEventListener("abort", onAbort); + reject(error); + }, + }); + window.webContents.send(DAPI_WIRE.CALL, { id, tool, args } satisfies DapiCall); + }); + } + + private track(window: BrowserWindow): void { + this.window = window; + const fail = (why: string) => () => this.failAll(why); + window.webContents.on("did-start-loading", fail("The app reloaded before replying")); + window.webContents.on("render-process-gone", fail("The app's renderer crashed")); + window.on("closed", () => { + if (this.window === window) this.window = null; + this.failAll("The app window closed before replying"); + }); + } + + private failAll(message: string): void { + const calls = [...this.inFlight.values()]; + this.inFlight.clear(); + for (const call of calls) call.reject(new Error(message)); + } + + // Resolves with the current window once it has finished loading. + private ready(timeoutMs = 30000): Promise { + const window = this.window; + if (!window || window.isDestroyed()) return Promise.reject(new Error("The app has no window")); + if (window.webContents.isCrashed()) return Promise.reject(new Error("The app's renderer crashed")); + if (!window.webContents.isLoading()) return Promise.resolve(window); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("The app did not become ready in time")); + }, timeoutMs); + const cleanup = () => { + clearTimeout(timer); + window.webContents.off("did-finish-load", onLoad); + window.webContents.off("did-fail-load", onFail); + }; + const onLoad = () => { + cleanup(); + resolve(window); + }; + const onFail = () => { + cleanup(); + reject(new Error("The app failed to load")); + }; + window.webContents.on("did-finish-load", onLoad); + window.webContents.on("did-fail-load", onFail); + }); + } +} diff --git a/apps/desktop/src/dapi/server.ts b/apps/desktop/src/dapi/server.ts new file mode 100644 index 00000000..5cd18734 --- /dev/null +++ b/apps/desktop/src/dapi/server.ts @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { chmodSync, existsSync, unlinkSync } from "node:fs"; +import { createServer } from "node:net"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { tools } from "@diffusionstudio/dapi"; +import { SOCKET_PATH, SocketTransport } from "@diffusionstudio/dapi/socket"; +import { mainHandlers } from "./handlers"; +import { present, toCallToolResult, toErrorResult } from "./present"; +import { RendererCalls } from "./renderer-calls"; + +import type { Server, Socket } from "node:net"; +import type { GenericTool, LogEntry, ToolName } from "@diffusionstudio/dapi"; +import type { MainContext, MainToolName } from "./handler"; + +export type DapiServerDeps = { + version: string; + /** The app's console buffer, for `logs` and `report`. */ + logs(): LogEntry[]; + /** Called once, on the first connection: an agent is driving, so the UI may step back. */ + onFirstConnection(): void; +}; + +/** + * The app's MCP server. Listens on the local socket; each connection gets its + * own MCP session over the catalog. Main-process tools run here; renderer + * tools are forwarded over IPC and their results presented (files written, + * small images inlined) before they go back out. + */ +export class DapiServer { + private readonly deps: DapiServerDeps; + private readonly renderer = new RendererCalls(); + private readonly sessions = new Set(); + private server: Server | null = null; + private connected = false; + + constructor(deps: DapiServerDeps) { + this.deps = deps; + for (const tool of tools) { + if (tool.runsIn === "main" && !(tool.name in mainHandlers)) { + throw new Error(`Main-process tool "${tool.name}" has no handler`); + } + } + } + + start(): void { + removeStaleSocket(); + this.renderer.start(); + this.server = createServer((socket) => void this.accept(socket)); + this.server.on("error", (error) => console.error("[dapi] server error:", error)); + this.server.listen(SOCKET_PATH, () => { + // Linux shares /tmp between users; the socket file's mode is the auth. + if (process.platform !== "win32") chmodSync(SOCKET_PATH, 0o600); + }); + } + + stop(): void { + for (const session of this.sessions) void session.close(); + this.sessions.clear(); + this.server?.close(); + this.server = null; + removeStaleSocket(); + } + + private async accept(socket: Socket): Promise { + if (!this.connected) { + this.connected = true; + this.deps.onFirstConnection(); + } + + const session = new McpServer({ name: "diffusion-studio", version: this.deps.version }); + for (const tool of tools) this.register(session, tool); + this.sessions.add(session); + session.server.onclose = () => this.sessions.delete(session); + try { + await session.connect(new SocketTransport(socket)); + } catch (error) { + console.error("[dapi] session failed to start:", error); + this.sessions.delete(session); + socket.destroy(); + } + } + + private register(session: McpServer, tool: GenericTool): void { + session.registerTool( + tool.name, + { title: tool.title, description: tool.description, inputSchema: tool.input, outputSchema: tool.output }, + async (args, extra) => { + try { + const result = + tool.runsIn === "main" + ? await this.runInMain(tool.name as MainToolName, args, extra.signal) + : await this.renderer.call(tool.name, args, extra.signal); + return toCallToolResult(await present(tool.name as ToolName, args, result)); + } catch (error) { + return toErrorResult(error); + } + }, + ); + } + + private runInMain(name: MainToolName, args: unknown, signal: AbortSignal): Promise { + const ctx: MainContext = { signal, logs: this.deps.logs, version: this.deps.version }; + // Each handler takes its own parsed args; the map's union type cannot + // express that pairing, so the call site widens. + return (mainHandlers[name] as (args: unknown, ctx: MainContext) => Promise)(args, ctx); + } +} + +/** + * A socket file left by a previous run (Unix). Safe to remove because the + * single-instance lock guarantees no other instance of ours is running. + */ +function removeStaleSocket(): void { + try { + if (process.platform !== "win32" && existsSync(SOCKET_PATH)) unlinkSync(SOCKET_PATH); + } catch { + // Best-effort. + } +} diff --git a/apps/desktop/src/edit-types.ts b/apps/desktop/src/edit-types.ts new file mode 100644 index 00000000..1523cc57 --- /dev/null +++ b/apps/desktop/src/edit-types.ts @@ -0,0 +1,140 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The shapes of a source edit and its outcome: the contract between the +// canvas (renderer) and the edit writer (main). Types only, with no Node +// imports, so the renderer's program can include it without Node's types. + +import type { InspectValue, PropValue, SerializedAssetRef } from "@diffusionstudio/jsx"; + +export type { PropValue, SerializedAssetRef }; + +/** + * A value an edit can carry: what a source spells as a literal, or a + * `generate.*` declaration in its wire form, spelled as the call that + * reproduces it (see `setProp`). + */ +export type EditValue = PropValue | SerializedAssetRef; + +export interface SourceContext { + /** Absolute path of the project folder. */ + dir: string; + /** Called with the project-relative path of every file written. */ + onWrite?: (file: string) => void; +} + +/** + * Overwrites props of the element named by `source` (a `SOURCE_ATTR` value), + * and — for a `` — what it says. `text` is its literal content, which is + * its children rather than a prop and so arrives on its own; an element that + * only says something new comes with no props at all. + */ +export interface SourceSet { + kind: "set"; + source: string; + props: Record; + text?: string; +} + +/** + * Adds `` under the element named by `parent`, in front of + * the child named by `before` or last. `source` is the pending name the canvas + * knows the new element by; the write answers with the real one in `ids`. + * A parent may itself be pending when it was inserted earlier in the same + * write. `text`, when present, is the element's literal text content + * (`Hello`); without it the element is written self-closing. + */ +export interface SourceInsert { + kind: "insert"; + source: string; + parent: string; + tag: string; + props: Record; + before?: string; + text?: string; +} + +/** + * Moves the element named by `source` under the one named by `parent`, in + * front of the child named by `before` or last. The element travels as it was + * written — its own text, re-indented for where it lands — so a move is the + * one edit that does not touch what an element says, only where it says it. + * Both ends must be in one file: an element cannot move into another module's + * JSX any more than a project could have put it there. + */ +export interface SourceMove { + kind: "move"; + source: string; + parent: string; + before?: string; +} + +/** + * Removes the element named by `source` from the file, and with it everything + * it contains: its children are its text, and go the way a move takes them + * along. Addressed like a move — an unnamed element is a position, and cutting + * text renumbers positions, so the element is found before anything is cut. + */ +export interface SourceRemove { + kind: "remove"; + source: string; +} + +/** + * One iteration of a loop as the canvas rendered it: for every composition + * element of the loop body, by its source, the props it came out with and (for + * a ``) its literal content — the values that were computed from the + * item, spelled as literals. `pending` is the name the canvas already knows + * that iteration's copy of the element by; the write answers with the real + * one in `ids`. The first iteration keeps the body's own names, so it carries + * none. + */ +export type SourceIteration = Record; text?: string; pending?: string }>; + +/** + * Replaces the ``/`` around the element named by `source` with + * one copy of its body per iteration, each spelling out what that iteration + * rendered. Nothing that comes after this in the same write can address a + * looped element (see `inLoop`): the loop is a recipe for elements, and a + * change to one of them means writing them down first. + */ +export interface SourceUnroll { + kind: "unroll"; + source: string; + iterations: SourceIteration[]; +} + +/** + * Overwrites the initializer of an `@inspect`-annotated top-level const (see + * @diffusionstudio/jsx's inspect). Addressed by file and variable name — a + * const's name is unique in its module, so no id is needed — and only written + * when the declaration still carries the annotation and holds a literal: an + * initializer someone rewrote into an expression is theirs again. + */ +export interface SourceVariable { + kind: "variable"; + file: string; + name: string; + value: InspectValue; +} + +export type SourceEdit = SourceSet | SourceInsert | SourceMove | SourceRemove | SourceUnroll | SourceVariable; + +export interface WriteResult { + /** Sources that could not be written, as `id` or `id (prop)`. */ + skipped: string[]; + /** + * Elements that earned a name in this write, as `old source id -> new one`. + * The canvas re-stamps its entities with these, so identity does not have to + * wait for a recompile. + */ + ids?: Record; + /** + * The loops this write unrolled, by the `source` of the `SourceUnroll` that + * asked. An unroll not listed here was declined, and the canvas takes its + * loop back (see the edit writer). + */ + unrolled?: string[]; + error?: string; +} diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.ts index b878d6de..f0f7dd70 100644 --- a/apps/desktop/src/edit.ts +++ b/apps/desktop/src/edit.ts @@ -11,7 +11,7 @@ import { IndentationText, Project, SyntaxKind } from "ts-morph"; import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, parseSource } from "@diffusionstudio/jsx"; -import type { InspectValue, PropValue, SerializedAssetRef } from "@diffusionstudio/jsx"; +import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx"; import type { ArrowFunction, FunctionExpression, @@ -24,136 +24,19 @@ import type { SourceFile, } from "ts-morph"; -export type { PropValue, SerializedAssetRef }; - -/** - * A value an edit can carry: what a source spells as a literal, or a - * `generate.*` declaration in its wire form, spelled as the call that - * reproduces it (see `setProp`). - */ -export type EditValue = PropValue | SerializedAssetRef; - -export interface SourceContext { - /** Absolute path of the project folder. */ - dir: string; - /** Called with the project-relative path of every file written. */ - onWrite?: (file: string) => void; -} - -/** - * Overwrites props of the element named by `source` (a `SOURCE_ATTR` value), - * and — for a `` — what it says. `text` is its literal content, which is - * its children rather than a prop and so arrives on its own; an element that - * only says something new comes with no props at all. - */ -export interface SourceSet { - kind: "set"; - source: string; - props: Record; - text?: string; -} - -/** - * Adds `` under the element named by `parent`, in front of - * the child named by `before` or last. `source` is the pending name the canvas - * knows the new element by; the write answers with the real one in `ids`. - * A parent may itself be pending when it was inserted earlier in the same - * write. `text`, when present, is the element's literal text content - * (`Hello`); without it the element is written self-closing. - */ -export interface SourceInsert { - kind: "insert"; - source: string; - parent: string; - tag: string; - props: Record; - before?: string; - text?: string; -} - -/** - * Moves the element named by `source` under the one named by `parent`, in - * front of the child named by `before` or last. The element travels as it was - * written — its own text, re-indented for where it lands — so a move is the - * one edit that does not touch what an element says, only where it says it. - * Both ends must be in one file: an element cannot move into another module's - * JSX any more than a project could have put it there. - */ -export interface SourceMove { - kind: "move"; - source: string; - parent: string; - before?: string; -} - -/** - * Removes the element named by `source` from the file, and with it everything - * it contains: its children are its text, and go the way a move takes them - * along. Addressed like a move — an unnamed element is a position, and cutting - * text renumbers positions, so the element is found before anything is cut. - */ -export interface SourceRemove { - kind: "remove"; - source: string; -} - -/** - * One iteration of a loop as the canvas rendered it: for every composition - * element of the loop body, by its source, the props it came out with and (for - * a ``) its literal content — the values that were computed from the - * item, spelled as literals. `pending` is the name the canvas already knows - * that iteration's copy of the element by; the write answers with the real - * one in `ids`. The first iteration keeps the body's own names, so it carries - * none. - */ -export type SourceIteration = Record; text?: string; pending?: string }>; - -/** - * Replaces the ``/`` around the element named by `source` with - * one copy of its body per iteration, each spelling out what that iteration - * rendered. Nothing that comes after this in the same write can address a - * looped element (see `inLoop`): the loop is a recipe for elements, and a - * change to one of them means writing them down first. - */ -export interface SourceUnroll { - kind: "unroll"; - source: string; - iterations: SourceIteration[]; -} - -/** - * Overwrites the initializer of an `@inspect`-annotated top-level const (see - * @diffusionstudio/jsx's inspect). Addressed by file and variable name — a - * const's name is unique in its module, so no id is needed — and only written - * when the declaration still carries the annotation and holds a literal: an - * initializer someone rewrote into an expression is theirs again. - */ -export interface SourceVariable { - kind: "variable"; - file: string; - name: string; - value: InspectValue; -} - -export type SourceEdit = SourceSet | SourceInsert | SourceMove | SourceRemove | SourceUnroll | SourceVariable; - -export interface WriteResult { - /** Sources that could not be written, as `id` or `id (prop)`. */ - skipped: string[]; - /** - * Elements that earned a name in this write, as `old source id -> new one`. - * The canvas re-stamps its entities with these, so identity does not have to - * wait for a recompile. - */ - ids?: Record; - /** - * The loops this write unrolled, by the `source` of the `SourceUnroll` that - * asked. An unroll not listed here was declined, and the canvas takes its - * loop back (see the edit writer). - */ - unrolled?: string[]; - error?: string; -} +export type * from "./edit-types"; +import type { + EditValue, + SourceContext, + SourceEdit, + SourceInsert, + SourceIteration, + SourceMove, + SourceRemove, + SourceUnroll, + SourceVariable, + WriteResult, +} from "./edit-types"; /** The opening half of a JSX element — where its attributes live. */ type JsxTag = JsxOpeningElement | JsxSelfClosingElement; diff --git a/apps/desktop/src/headless.ts b/apps/desktop/src/headless.ts new file mode 100644 index 00000000..b695807e --- /dev/null +++ b/apps/desktop/src/headless.ts @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Whether an agent has connected since launch. Once one has, the UI steps +// back: file writes from tools are not echoed as edits, and the window may +// stay hidden. Set by the MCP server on its first connection; read wherever +// behaviour depends on who is driving. + +let headless = false; + +export function isHeadless(): boolean { + return headless; +} + +export function enableHeadless(): void { + headless = true; +} diff --git a/apps/desktop/src/main-channels.ts b/apps/desktop/src/main-channels.ts index 9fd04403..fd88916d 100644 --- a/apps/desktop/src/main-channels.ts +++ b/apps/desktop/src/main-channels.ts @@ -7,10 +7,10 @@ // through these via an envelope carrying the logical MAIN_CHANNELS name and // (for requests) a UUID for correlation. // -// CLI traffic uses a separate wire pair (CLI_WIRE in @diffusionstudio/cli/protocol); -// main forwards it opaquely without inspecting channel names. +// Tool calls from the MCP server use their own wire (DAPI_WIRE in +// @diffusionstudio/dapi), in the other direction: main asks, the renderer answers. import type { LogEntry, ScreenshotResult } from "@diffusionstudio/dapi"; -import type { SourceEdit, WriteResult } from "./edit"; +import type { SourceEdit, WriteResult } from "./edit-types"; export const MAIN_WIRE = { REQUEST: "main:request", diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index bd79dd7c..23039413 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -8,7 +8,8 @@ import { mkdir, open, unlink } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import type { FileHandle } from "node:fs/promises"; import { updateElectronApp } from "update-electron-app"; -import { startCliServer, stopCliServer, isHeadless } from "./cli-server"; +import { DapiServer } from "./dapi/server"; +import { enableHeadless, isHeadless } from "./headless"; import { installCli, isCliInstalled } from "./cli-install"; import { healSkillsLinks, installSkills, isSkillsInstalled } from "./skills-install"; import { trackInstall } from "./analytics"; @@ -97,6 +98,19 @@ const pendingDeepLinks = new Map(); const LOG_BUFFER_MAX = 2000; const logBuffer: LogEntry[] = []; +// The MCP server agents and the dapi CLI talk to. Started once the app is +// ready; the first connection switches the UI into headless mode. +const dapi = new DapiServer({ + version: app.getVersion(), + logs: () => logBuffer, + onFirstConnection() { + enableHeadless(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainBridge.emit(mainWindow, MAIN_CHANNELS.HEADLESS_MODE, { active: true }); + } + }, +}); + function pushLog(level: LogEntry["level"], message: string, source: string) { logBuffer.push({ ts: Date.now(), level, message, source }); if (logBuffer.length > LOG_BUFFER_MAX) logBuffer.shift(); @@ -361,7 +375,7 @@ if (app.requestSingleInstanceLock()) { const url = findProtocolUrl(process.argv); if (url) deliverDeepLink(url); - startCliServer(); + dapi.start(); healSkillsLinks(); trackInstall(); createWindow(!isHiddenLaunch(process.argv)); @@ -369,7 +383,7 @@ if (app.requestSingleInstanceLock()) { app.on("before-quit", () => { unwatchAll(); - stopCliServer(); + dapi.stop(); }); app.on("window-all-closed", () => { diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 60e10afe..1793687d 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -4,16 +4,17 @@ import { contextBridge, ipcRenderer, webUtils } from "electron"; import { MAIN_WIRE } from "./main-channels"; -import { CLI_WIRE } from "@diffusionstudio/cli/protocol"; +import { DAPI_WIRE } from "@diffusionstudio/dapi"; import type { IpcRendererEvent } from "electron"; -const ALLOWED_RENDERER_TO_MAIN: ReadonlySet = new Set([MAIN_WIRE.REQUEST]); +const ALLOWED_RENDERER_TO_MAIN: ReadonlySet = new Set([MAIN_WIRE.REQUEST, DAPI_WIRE.REPLY]); const ALLOWED_MAIN_TO_RENDERER: ReadonlySet = new Set([ MAIN_WIRE.RESPONSE, MAIN_WIRE.EVENT, - CLI_WIRE.CONNECT, + DAPI_WIRE.CALL, + DAPI_WIRE.CANCEL, ]); contextBridge.exposeInMainWorld("desktop", { diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts index 533fffb9..d7303b25 100644 --- a/apps/desktop/src/projects.ts +++ b/apps/desktop/src/projects.ts @@ -14,7 +14,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; import type { PluginItem, TransformOptions } from "@babel/core"; import type { BuildOptions, Plugin } from "esbuild"; -import { isHeadless } from "./cli-server"; +import { isHeadless } from "./headless"; import { mainBridge } from "./main-manager"; import { MAIN_CHANNELS } from "./main-channels"; import { applyEdits, editLabel, stampProject } from "./edit"; diff --git a/apps/web/package.json b/apps/web/package.json index a60ab03c..929f6a3d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -15,7 +15,6 @@ "dependencies": { "@diffusionstudio/api-contract": "^0.1.0", "@diffusionstudio/assets": "*", - "@diffusionstudio/cli": "*", "@diffusionstudio/encoder": "*", "@diffusionstudio/jsx": "*", "@diffusionstudio/koota-solid": "*", diff --git a/apps/web/src/context/render.ts b/apps/web/src/context/render.ts index ab2eca7b..b7003a9d 100644 --- a/apps/web/src/context/render.ts +++ b/apps/web/src/context/render.ts @@ -32,6 +32,8 @@ export type RenderOverlayState = { remaining?: { minutes: number; seconds: number }; }; +const PROGRESS_LOG_STEP = 2; + const [overlay, setOverlay] = createSignal(null); let cancelActive: (() => void) | undefined; @@ -78,6 +80,13 @@ export async function renderScene( cancelActive = undefined; setOverlay({ config, width, height, duration, progress: 0, remaining: undefined }); + let logged = -1; + const logProgress = (percent: number) => { + if (percent - logged < PROGRESS_LOG_STEP && percent < 100) return; + logged = percent; + console.info(`[export] ${percent}%`); + }; + engine.stop(); let capture: Capture | undefined; @@ -94,6 +103,7 @@ export async function renderScene( comment: `Made with Diffusion Studio v${version}`, onProgress(p) { const percent = Math.round((p.progress / p.total) * 100); + logProgress(percent); setOverlay((prev) => prev ? { diff --git a/apps/web/src/dapi/api.tsx b/apps/web/src/dapi/api.tsx index 611f2246..f2d9c8ca 100644 --- a/apps/web/src/dapi/api.tsx +++ b/apps/web/src/dapi/api.tsx @@ -14,12 +14,12 @@ import { openProjectFolder } from "@/projects"; import { projectRoute } from "@/hooks/use-project-route"; import { useFullscreenState } from "@/hooks/use-fullscreen-state"; import { assert } from "@/utils/common"; -import { cliBridge } from "./bridge"; +import { toolBridge } from "./bridge"; import { handlers } from "./handlers"; import { editorSession, requireEditorSession, setEditorSession } from "./session"; import type { JSX, Accessor } from "solid-js"; -import type { ToolContext } from "./handler"; +import type { ContextFactory } from "./bridge"; type EditorApiContextValue = { isFullscreen: Accessor; @@ -39,7 +39,7 @@ export function EditorApi() { const navigate = useNavigate(); const auth = useAuth(); - const context = (signal: AbortSignal): ToolContext => ({ + const context: ContextFactory = (signal) => ({ session: editorSession, requireSession: requireEditorSession, signal, @@ -58,7 +58,7 @@ export function EditorApi() { }, }); - onCleanup(cliBridge.register(handlers, context)); + onCleanup(toolBridge.register(handlers, context)); return null; } diff --git a/apps/web/src/dapi/bridge.ts b/apps/web/src/dapi/bridge.ts index 2db5a591..c640bb56 100644 --- a/apps/web/src/dapi/bridge.ts +++ b/apps/web/src/dapi/bridge.ts @@ -2,48 +2,31 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { parseToolArgs } from "@diffusionstudio/dapi"; -import { CLI_WIRE, encodeReply } from "@diffusionstudio/cli/protocol"; +import { DAPI_WIRE, isDapiError } from "@diffusionstudio/dapi"; -import type { CliHandshake, CliReply, CliRequest } from "@diffusionstudio/cli/protocol"; +import type { DapiCall, DapiCancel, DapiReply } from "@diffusionstudio/dapi"; import type { Handlers, ServedToolName, ToolContext } from "./handler"; -type Pending = { req: CliRequest; ws: WebSocket }; - /** Builds the per-call context; the bridge supplies the abort signal. */ export type ContextFactory = (signal: AbortSignal) => ToolContext; /** - * Answers tool calls from the CLI. Each CLI command hosts a short-lived - * WebSocket server; main relays only the connect info (CLI_WIRE.CONNECT) and - * we dial the CLI directly, so payloads never pass through main. One request - * and one reply per connection. - * - * The bridge is transport: it validates arguments against the catalog, runs - * the handler, and encodes the reply. Requests arriving before the handlers - * register (page bootstrap) are held and answered on registration. + * Answers renderer tools for the MCP server in main. Main validated the + * arguments against the catalog before sending them; this side runs the + * handler and replies, honouring cancels. Calls that arrive before the + * handlers register (page bootstrap) are held and answered on registration. */ -class CliBridge { +class ToolBridge { private handlers: Handlers | null = null; private context: ContextFactory | null = null; - private held: Pending[] = []; + private held: DapiCall[] = []; + private readonly inFlight = new Map(); constructor() { - // Bind eagerly so CONNECT arrivals during page bootstrap are caught - // rather than silently dropped before the handlers register. - window.desktop?.on(CLI_WIRE.CONNECT, (payload) => { - const { port, token } = payload as CliHandshake; - const ws = new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`); - ws.onmessage = (event) => { - try { - const req = JSON.parse(event.data as string) as CliRequest; - void this.dispatch({ req, ws }); - } catch (err) { - console.error("[cli-bridge] malformed CLI request", err); - ws.close(); - } - }; - }); + // Bind eagerly so calls during page bootstrap are caught rather than + // silently dropped before the handlers register. + window.desktop?.on(DAPI_WIRE.CALL, (payload) => void this.dispatch(payload as DapiCall)); + window.desktop?.on(DAPI_WIRE.CANCEL, (payload) => this.inFlight.get((payload as DapiCancel).id)?.abort()); } register(handlers: Handlers, context: ContextFactory): () => void { @@ -51,7 +34,7 @@ class CliBridge { this.context = context; const held = this.held; this.held = []; - for (const pending of held) void this.dispatch(pending); + for (const call of held) void this.dispatch(call); return () => { if (this.handlers === handlers) { this.handlers = null; @@ -60,41 +43,31 @@ class CliBridge { }; } - private async dispatch(pending: Pending): Promise { - const { req, ws } = pending; - // Liveness, answered by the transport itself: `open` waits on it after - // launching the app, before any handler exists to answer anything else. - if (req.path === "ping") { - ws.send(encodeReply({ ok: true, data: undefined })); - return; - } + private async dispatch(call: DapiCall): Promise { if (!this.handlers || !this.context) { - this.held.push(pending); + this.held.push(call); return; } const controller = new AbortController(); - ws.addEventListener("close", () => controller.abort(), { once: true }); + this.inFlight.set(call.id, controller); - let reply: CliReply; + let reply: DapiReply; try { - const name = req.path as ServedToolName; - const handler = this.handlers[name]; - if (!handler) throw new Error(`Unknown tool "${req.path}"`); - const args = parseToolArgs(name, req.input); + const handler = this.handlers[call.tool as ServedToolName]; + if (!handler) throw new Error(`The app has no handler for "${call.tool}"`); // Every handler takes its own parsed args; the map's union type cannot // express that pairing, so the call site widens. - const data = await (handler as (args: unknown, ctx: ToolContext) => Promise)(args, this.context(controller.signal)); - reply = { ok: true, data }; - } catch (err) { - reply = { ok: false, error: (err as Error).message }; - } - try { - ws.send(encodeReply(reply)); - } catch (err) { - ws.send(encodeReply({ ok: false, error: `Failed to serialize reply for ${req.path}: ${(err as Error).message}` })); + const run = handler as (args: unknown, ctx: ToolContext) => Promise; + reply = { id: call.id, ok: true, data: await run(call.args, this.context(controller.signal)) }; + } catch (error) { + const message = (error as Error).message; + reply = { id: call.id, ok: false, error: isDapiError(error) ? { code: error.code, message } : { message } }; + } finally { + this.inFlight.delete(call.id); } + if (!controller.signal.aborted) window.desktop?.send(DAPI_WIRE.REPLY, reply); } } -export const cliBridge = new CliBridge(); +export const toolBridge = new ToolBridge(); diff --git a/apps/web/src/dapi/handler.ts b/apps/web/src/dapi/handler.ts index 1a7f67ab..22237088 100644 --- a/apps/web/src/dapi/handler.ts +++ b/apps/web/src/dapi/handler.ts @@ -17,7 +17,7 @@ export type ToolContext = { session: Accessor; /** The session, or a `no-project` error the caller can act on. */ requireSession(): EditorSession; - /** Fires when the caller goes away before the reply. */ + /** Fires when the caller cancels or goes away before the reply. */ signal: AbortSignal; /** What only the app shell can do: navigate, and know who is signed in. */ app: { @@ -30,11 +30,7 @@ export type ToolContext = { export type ToolHandler = (args: ToolArgs, ctx: ToolContext) => Promise>; -/** - * The tools the renderer answers. `logs` is a main-process tool in the - * catalog and is forwarded from here until main hosts the server itself; - * `fonts`, `fetch` and `report` have no renderer side at all. - */ -export type ServedToolName = Exclude; +/** The tools the renderer answers; the rest run in the main process. */ +export type ServedToolName = Exclude; export type Handlers = { readonly [N in ServedToolName]: ToolHandler }; diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.ts index d75b349c..ff5d2bb2 100644 --- a/apps/web/src/dapi/handlers/context.ts +++ b/apps/web/src/dapi/handlers/context.ts @@ -21,7 +21,7 @@ export const context: ToolHandler<"context"> = async (_, ctx) => { const rootDir = await getProjectsRoot(); const open = ctx.session(); - if (!open) return { rootDir, projectDir: null }; + if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], generations: [] }; const { world, project } = open; const frameRate = world.get(FrameRate)?.value || 30; diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.ts index 2277306b..9aa95a0b 100644 --- a/apps/web/src/dapi/handlers/index.ts +++ b/apps/web/src/dapi/handlers/index.ts @@ -10,7 +10,6 @@ import { exportScene } from "./export"; import { models } from "./models"; import { voices } from "./voices"; import { whoami } from "./whoami"; -import { logs } from "./logs"; import { screenshot } from "./screenshot"; import { mediaProbe } from "./media-probe"; import { mediaGrab } from "./media-grab"; @@ -31,7 +30,6 @@ export const handlers: Handlers = { models, voices, whoami, - logs, screenshot, media_probe: mediaProbe, media_grab: mediaGrab, diff --git a/apps/web/src/dapi/handlers/logs.ts b/apps/web/src/dapi/handlers/logs.ts deleted file mode 100644 index cdb909f8..00000000 --- a/apps/web/src/dapi/handlers/logs.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { mainBridge } from "@/lib/ipc"; -import { MAIN_CHANNELS } from "@desktop/main-channels"; - -import type { LogLevel } from "@diffusionstudio/dapi"; -import type { ToolHandler } from "../handler"; - -const LEVEL_RANK: Record = { debug: 0, info: 1, warning: 2, error: 3 }; - -// Main owns the buffer; this is a forward until main answers tools itself. -export const logs: ToolHandler<"logs"> = async ({ tail, level }) => { - let entries = await mainBridge.call(MAIN_CHANNELS.LOGS_GET, undefined); - if (level !== undefined) { - const min = LEVEL_RANK[level]; - entries = entries.filter((e) => LEVEL_RANK[e.level] >= min); - } - if (tail !== undefined) entries = entries.slice(-tail); - return entries; -}; diff --git a/apps/web/src/dapi/handlers/models.ts b/apps/web/src/dapi/handlers/models.ts index b9d575b9..5209511a 100644 --- a/apps/web/src/dapi/handlers/models.ts +++ b/apps/web/src/dapi/handlers/models.ts @@ -31,5 +31,5 @@ export const models: ToolHandler<"models"> = async ({ type }) => { if (!type || type === "audio") { for (const { id, name } of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) out.push({ type: "audio", id, name }); } - return out; + return { models: out }; }; diff --git a/apps/web/src/dapi/handlers/voices.ts b/apps/web/src/dapi/handlers/voices.ts index e589b0c0..65fd8982 100644 --- a/apps/web/src/dapi/handlers/voices.ts +++ b/apps/web/src/dapi/handlers/voices.ts @@ -6,5 +6,6 @@ import { PROMPT_INPUT_VOICE_OPTIONS } from "@/components/genai/config"; import type { ToolHandler } from "../handler"; -export const voices: ToolHandler<"voices"> = async () => - PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ id: v.value, label: v.label, description: v.description })); +export const voices: ToolHandler<"voices"> = async () => ({ + voices: PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ id: v.value, label: v.label, description: v.description })), +}); diff --git a/apps/web/src/dapi/handlers/whoami.ts b/apps/web/src/dapi/handlers/whoami.ts index 23022db5..a23daaa7 100644 --- a/apps/web/src/dapi/handlers/whoami.ts +++ b/apps/web/src/dapi/handlers/whoami.ts @@ -4,4 +4,4 @@ import type { ToolHandler } from "../handler"; -export const whoami: ToolHandler<"whoami"> = async (_, ctx) => ctx.app.user(); +export const whoami: ToolHandler<"whoami"> = async (_, ctx) => ({ user: ctx.app.user() }); diff --git a/docs/mcp-server.md b/docs/mcp-server.md index d32644b3..94d0a1dd 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -17,7 +17,7 @@ The point is not MCP for its own sake. The point is that the API gets one defini - One definition of the API: tool names, input schemas, descriptions and result shapes, owned by the app and served to clients at runtime. - Agents reach the app with a one-line registration and no PATH install. -- Long operations (export, generation, `listen`) report progress and can be cancelled, without client-side timeouts. +- Long operations (export, generation, `listen`) can be cancelled, and their progress is visible through `logs`. - The CLI binary stays useful for humans, scripts and CI, and no longer has to be built against a specific router type. - Keep the renderer handlers and their behaviour. This is a transport and surface change, not a rewrite of `capture`, `check`, `export` or the media handlers. @@ -141,14 +141,13 @@ for (const tool of catalog) { **Main-to-renderer calls.** The current bridge (`apps/desktop/src/main-manager.ts` and `MainBridge` in `apps/web/src/lib/ipc.ts`) carries requests from the renderer to main and events from main to the renderer. It has no request in the other direction, so this adds one, with three IPC messages: -- `dapi:call` from main: `{ callId, tool, input }` -- `dapi:progress` from renderer: `{ callId, progress, total?, message? }` -- `dapi:reply` from renderer: `{ callId, ok, data | error }` -- `dapi:cancel` from main: `{ callId }` +- `dapi:call` from main: `{ id, tool, args }` +- `dapi:reply` from renderer: `{ id, ok, data | error }` +- `dapi:cancel` from main: `{ id }` Main keeps a map of in-flight calls. Before sending it waits for the renderer to finish loading, which `waitForRendererReady` already does. In the renderer, `CliBridge` loses its WebSocket code and keeps its dispatch and hold-until-a-handler-registers logic, now fed by `dapi:call`. The renderer registers a plain `Record` instead of a tRPC router; each handler receives the already-validated input, an `AbortSignal`, and a `report(progress)` callback. tRPC and its `cli-rpc.ts` helpers go away. -**Progress and cancellation.** When the client passes a `progressToken`, main turns `dapi:progress` into `notifications/progress`. When the client sends `notifications/cancelled`, main sends `dapi:cancel` and the handler's `AbortSignal` fires. Client-side timeouts disappear; the client decides how long it is willing to wait. `export` reports encoded frames; `media_listen` and generation-backed work report whatever stage they are in; the others report nothing and that is fine. +**Progress and cancellation.** When the client sends `notifications/cancelled`, main sends `dapi:cancel` and the handler's `AbortSignal` fires. Progress needs no channel of its own: the export path logs its percentage to the console in 2% steps, the app buffers console output, and an agent waiting on an export polls `logs`. That works the same for an in-app export and a tool export, and keeps the call path to one request and one reply. **Results.** Every tool returns `structuredContent` matching its output schema, plus a text block with the same JSON so clients that ignore structured content still get it. Tools that produce images (`capture`, `media_grab`, `media_filmstrip`, `media_waveform`, `screenshot`) keep today's behaviour of writing PNGs to an output directory (an `output` input, defaulting to a fresh directory under the temp dir) and returning the paths. In addition, they inline `image` content blocks when the result is small: at most four images and none over about a megabyte. A contact sheet of a few positions therefore arrives in the agent's context immediately, while an uncapped hundred-frame grab arrives as a directory the agent reads selectively, which is the behaviour the current CLI was designed for. @@ -193,9 +192,9 @@ Removed from the CLI package: `cli-client.ts`, `cli-channels.ts`, the tRPC and ` The CLI ships inside the app bundle, so there is no compatibility window to maintain between old CLIs and new apps. This can land as one feature branch in ordered steps, each of which builds and type-checks on its own. 1. **Catalog.** Create `packages/dapi` with the tool catalog, `Time`, and `DapiError`, plus tests. Point the renderer handlers' request and result types at it. Delete `apps/cli/src/cli-channels.ts`; `apps/cli/src/protocol.ts` keeps only the handshake and envelope types of the current transport. No behaviour change. *Landed on `mcp-support`.* -2. **Renderer.** Replace the tRPC router in `api.tsx` with one handler per tool behind a uniform `(args, ctx)` signature, validated by the catalog schemas, with an `AbortSignal` per call. Handlers return bytes; the WebSocket transport carries them as tagged base64 until step 3 replaces it. The CLI drops tRPC for a typed `call(name, input)` over the same catalog, so it keeps working at every step and its import of `apps/web` is gone. Progress reporting waits for step 3, with the transport that carries it. *Landed on `mcp-support`.* -3. **Main.** Add `mcp-server.ts` with the socket transport, catalog registration, the in-flight map, result presentation, and the `logs`, `fonts`, `fetch` and `report` handlers. Delete `cli-server.ts`. Verify with the MCP Inspector against the socket. -4. **CLI.** Rewrite `apps/cli` as `mcp`, `open`, `call` and the wrappers. Drop tRPC and `ws`. Update `cli-install.ts` and the settings UI to offer MCP registration. +2. **Renderer.** Replace the tRPC router in `api.tsx` with one handler per tool behind a uniform `(args, ctx)` signature, validated by the catalog schemas, with an `AbortSignal` per call. Handlers return bytes; the WebSocket transport carries them as tagged base64 until step 3 replaces it. The CLI drops tRPC for a typed `call(name, input)` over the same catalog, so it keeps working at every step and its import of `apps/web` is gone. *Landed on `mcp-support`.* +3. **Main.** Add `dapi/server.ts` with catalog registration, `dapi/renderer-calls.ts` with the in-flight map, `dapi/present.ts` for files and inline images, and the `logs`, `fonts`, `fetch` and `report` handlers. The socket transport lives in `@diffusionstudio/dapi/socket`, shared with the CLI, which becomes an MCP client over it in this step so it keeps working. Delete `cli-server.ts`. *Landed on `mcp-support`; verified by an in-process client/server round trip, not yet against the running app.* +4. **CLI.** Add `dapi mcp` and `dapi call`, and generate the wrappers from the catalog. Update `cli-install.ts` and the settings UI to offer MCP registration. 5. **Docs.** Update the skill and regenerate `reference/`. Update the installation reference the skill points to. 6. **Cleanup.** Remove `CLI_WIRE`, the handshake types, and the `./protocol` and `./channels` exports from the CLI package. diff --git a/package-lock.json b/package-lock.json index 23f685b9..296b1771 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,10 +25,10 @@ "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", "@diffusionstudio/dapi": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", - "esbuild": "^0.28.1", - "ws": "^8.18.3" + "esbuild": "^0.28.1" }, "bin": { "dapi": "dist/index.js" @@ -36,7 +36,6 @@ "devDependencies": { "@types/babel__core": "^7.20.5", "@types/node": "^24.10.1", - "@types/ws": "^8.18.1", "typescript": "~5.9.3" } }, @@ -47,9 +46,9 @@ "dependencies": { "@babel/core": "^7.29.7", "@babel/preset-typescript": "^7.27.1", - "@diffusionstudio/cli": "*", "@diffusionstudio/dapi": "*", "@diffusionstudio/jsx": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "babel-preset-solid": "^1.9.12", "esbuild": "^0.28.1", "nanoid": "^6.0.1", @@ -65,7 +64,8 @@ "@types/babel__core": "^7.20.5", "@types/node": "^24.10.1", "electron": "^43.1.1", - "typescript": "~5.9.3" + "typescript": "~5.9.3", + "vitest": "^4.0.0" } }, "apps/desktop/node_modules/nanoid": { @@ -93,7 +93,6 @@ "dependencies": { "@diffusionstudio/api-contract": "^0.1.0", "@diffusionstudio/assets": "*", - "@diffusionstudio/cli": "*", "@diffusionstudio/dapi": "*", "@diffusionstudio/encoder": "*", "@diffusionstudio/jsx": "*", @@ -2007,6 +2006,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -2473,6 +2484,85 @@ "node": ">= 12.13.0" } }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -4239,16 +4329,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/ws": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", - "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/yauzl": { "version": "2.10.3", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", @@ -4858,6 +4938,64 @@ "dev": true, "license": "ISC" }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/acorn": { "version": "8.17.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", @@ -5272,6 +5410,59 @@ "dev": true, "license": "MIT" }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -5405,6 +5596,15 @@ "dev": true, "license": "MIT" }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/cacache": { "version": "16.1.3", "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz", @@ -5547,7 +5747,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -5561,7 +5760,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -5933,12 +6131,69 @@ "dev": true, "license": "MIT" }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "license": "MIT" }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/cross-dirname": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/cross-dirname/-/cross-dirname-0.1.0.tgz", @@ -5950,7 +6205,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -6211,6 +6465,15 @@ "dev": true, "license": "MIT" }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", @@ -6336,7 +6599,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -6354,6 +6616,12 @@ "dev": true, "license": "MIT" }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, "node_modules/electron": { "version": "43.1.1", "resolved": "https://registry.npmjs.org/electron/-/electron-43.1.1.tgz", @@ -6450,6 +6718,15 @@ "license": "MIT", "optional": true }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/encoding": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", @@ -6542,7 +6819,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6552,7 +6828,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6569,7 +6844,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -6636,6 +6910,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -6897,6 +7177,15 @@ "url": "https://github.com/eta-dev/eta?sponsor=1" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/eventemitter3": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", @@ -6914,6 +7203,27 @@ "node": ">=0.8.x" } }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/execa": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", @@ -7030,6 +7340,84 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -7086,7 +7474,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -7137,7 +7524,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", - "dev": true, "funding": [ { "type": "github", @@ -7224,9 +7610,30 @@ "node": ">=8" } }, - "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "license": "MIT", @@ -7297,6 +7704,24 @@ "imul": "^1.0.0" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -7377,7 +7802,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7443,7 +7867,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -7501,7 +7924,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -7638,7 +8060,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7707,7 +8128,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7720,7 +8140,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7729,6 +8148,15 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "4.13.5", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.5.tgz", + "integrity": "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, "node_modules/hosted-git-info": { "version": "2.8.9", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", @@ -7762,6 +8190,26 @@ "dev": true, "license": "BSD-2-Clause" }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -7959,7 +8407,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/ini": { @@ -7993,12 +8440,20 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 12" } }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -8137,6 +8592,12 @@ "node": ">=0.12.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", @@ -8224,7 +8685,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, "license": "ISC" }, "node_modules/jest-worker": { @@ -8268,6 +8728,15 @@ "jiti": "lib/jiti-cli.mjs" } }, + "node_modules/jose": { + "version": "6.2.10", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", + "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -8323,6 +8792,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, "node_modules/json-stable-stringify": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz", @@ -9121,7 +9596,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -9134,6 +9608,19 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/mediabunny": { "version": "1.50.6", "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.50.6.tgz", @@ -9183,6 +9670,18 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -9218,7 +9717,6 @@ "version": "1.54.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -9689,6 +10187,27 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -9713,11 +10232,22 @@ "node": ">=12.20.0" } }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -10009,6 +10539,15 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/patch-package": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.1.tgz", @@ -10079,7 +10618,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -10092,6 +10630,16 @@ "dev": true, "license": "MIT" }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/path-type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", @@ -10163,6 +10711,15 @@ "node": ">=0.10.0" } }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/plist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.1.tgz", @@ -10300,6 +10857,19 @@ "node": ">=10" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", @@ -10321,6 +10891,22 @@ "node": ">=6" } }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -10367,6 +10953,50 @@ "murmur-32": "^0.1.0 || ^0.2.0" } }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/read-binary-file-arch": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/read-binary-file-arch/-/read-binary-file-arch-1.0.6.tgz", @@ -10525,7 +11155,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10734,6 +11363,22 @@ "fsevents": "~2.3.2" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -10783,7 +11428,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true, "license": "MIT" }, "node_modules/sax": { @@ -10874,6 +11518,48 @@ "license": "MIT", "optional": true }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/serialize-error": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", @@ -10926,6 +11612,25 @@ "seroval": "^1.0" } }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -10944,11 +11649,16 @@ "node": ">= 0.4" } }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -10961,12 +11671,83 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -11231,6 +12012,15 @@ "integrity": "sha512-hNKz8phvYLPEcRkeG1rsGmV5ChMjKDAWU7/OJJdDErPBNChQXxCo3WZurGpnWc6gZhAzEPFad1aVgyOANH1sMw==", "license": "MIT" }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/std-env": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", @@ -11688,6 +12478,15 @@ "node": ">=8.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -11789,6 +12588,53 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/typed-binary": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/typed-binary/-/typed-binary-4.3.3.tgz", @@ -11897,6 +12743,15 @@ "node": ">= 0.4.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/unplugin": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.3.0.tgz", @@ -12083,6 +12938,15 @@ "spdx-expression-parse": "^3.0.0" } }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/vite": { "version": "7.3.6", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", @@ -12497,7 +13361,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -12587,30 +13450,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, - "node_modules/ws": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", - "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xmlbuilder": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", @@ -12771,6 +13612,15 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + }, "packages/assets": { "name": "@diffusionstudio/assets", "version": "0.1.0", @@ -12789,6 +13639,7 @@ "license": "MPL-2.0", "dependencies": { "@diffusionstudio/jsx": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "zod": "^4.4.3" }, "devDependencies": { diff --git a/packages/dapi/package.json b/packages/dapi/package.json index 6a10c94e..b25e0166 100644 --- a/packages/dapi/package.json +++ b/packages/dapi/package.json @@ -20,6 +20,7 @@ ], "dependencies": { "@diffusionstudio/jsx": "*", + "@modelcontextprotocol/sdk": "^1.30.0", "zod": "^4.4.3" }, "devDependencies": { diff --git a/packages/dapi/src/catalog.test.ts b/packages/dapi/src/catalog.test.ts index c0641573..844c286e 100644 --- a/packages/dapi/src/catalog.test.ts +++ b/packages/dapi/src/catalog.test.ts @@ -26,10 +26,11 @@ describe("catalog", () => { } }); - it("converts every schema to JSON Schema without throwing (bytes have no JSON form and pass as any)", () => { + it("converts every input and output to JSON Schema, as MCP tools/list does", () => { for (const tool of catalog) { expect(() => z.toJSONSchema(tool.input, { io: "input" }), `${tool.name} input`).not.toThrow(); - expect(() => z.toJSONSchema(tool.output, { unrepresentable: "any" }), `${tool.name} output`).not.toThrow(); + expect(() => z.toJSONSchema(tool.output), `${tool.name} output`).not.toThrow(); + expect(tool.output, `${tool.name} output must be an object for structured content`).toBeInstanceOf(z.ZodObject); } }); diff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.ts index 0db7d724..39c70d60 100644 --- a/packages/dapi/src/catalog.ts +++ b/packages/dapi/src/catalog.ts @@ -3,7 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import type { z } from "zod"; -import type { Tool } from "./tool"; +import type { GenericTool } from "./tool"; import { open } from "./tools/open"; import { context } from "./tools/context"; @@ -60,7 +60,14 @@ export type ToolByName = Extract; export type ToolInput = z.input["input"]>; /** What a handler receives: parsed, defaults applied. */ export type ToolArgs = z.output["input"]>; -export type ToolResult = z.output["output"]>; +/** What a caller receives: the tool's structured content. */ +export type ToolOutput = z.output["output"]>; +/** What a handler returns; the output, unless the tool declares a `result`. */ +export type ToolResult = ToolByName extends { result?: infer R } + ? R extends z.ZodType + ? z.output + : ToolOutput + : ToolOutput; const byName = new Map(catalog.map((tool) => [tool.name, tool])); @@ -72,5 +79,5 @@ export function isToolName(name: string): name is ToolName { return byName.has(name); } -/** The catalog as the generic `Tool` shape, for code that iterates it. */ -export const tools: readonly Tool[] = catalog; +/** The catalog with each tool's specifics erased, for code that iterates it. */ +export const tools: readonly GenericTool[] = catalog; diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.ts index da23f6c8..d6647796 100644 --- a/packages/dapi/src/index.ts +++ b/packages/dapi/src/index.ts @@ -8,10 +8,10 @@ import type { z } from "zod"; export { defineTool } from "./tool"; -export type { Tool, RunsIn } from "./tool"; +export type { Tool, GenericTool, RunsIn } from "./tool"; export { catalog, tools, toolByName, isToolName } from "./catalog"; -export type { AnyTool, ToolName, ToolByName, ToolInput, ToolArgs, ToolResult } from "./catalog"; +export type { AnyTool, ToolName, ToolByName, ToolInput, ToolArgs, ToolOutput, ToolResult } from "./catalog"; export { Time, NonNegativeTime, TIME_FORMS } from "./time"; export type { TimeInput } from "./time"; @@ -20,13 +20,15 @@ export { DapiError, isDapiError } from "./errors"; export type { DapiErrorCode } from "./errors"; export { MAX_FRAMES_PER_SHEET, Bytes } from "./schemas"; -export { parseToolArgs } from "./validate"; + +export { DAPI_WIRE } from "./ipc"; +export type { DapiCall, DapiCancel, DapiReply } from "./ipc"; export { FRAME_CAP } from "./tools/media-grab"; export { ISSUE_LOG_TAIL } from "./tools/report"; // Named request and result types, for handlers that spell out their // signature. Each is the parsed (output) side of the tool's schema. -import type { LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas"; +import type { ImageRef as ImageRefSchema, LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas"; import type { GenerationRow as GenerationRowType } from "./tools/context"; import type { CheckIssue as CheckIssueSchema, CheckIssueCode as CheckIssueCodeSchema } from "./tools/check"; import type { ExportFormat as ExportFormatSchema, ExportSettings as ExportSettingsSchema } from "./tools/export"; @@ -35,11 +37,12 @@ import type { VoiceInfo as VoiceInfoSchema } from "./tools/voices"; import type { FrameQuality as FrameQualitySchema } from "./tools/media-grab"; import type { TranscriptSegment as TranscriptSegmentSchema, TranscriptWord as TranscriptWordSchema } from "./tools/media-transcribe"; import type { FontFamily as FontFamilySchema } from "./tools/fonts"; -import type { ToolArgs, ToolResult } from "./catalog"; +import type { ToolArgs, ToolOutput, ToolResult } from "./catalog"; export type LogLevel = z.output; export type LogEntry = z.output; export type TimecodedImage = z.output; +export type ImageRef = z.output; export type GenerationRow = GenerationRowType; export type CheckIssueCode = z.output; export type CheckIssue = z.output; @@ -54,7 +57,7 @@ export type FontFamily = z.output; export type OpenRequest = ToolArgs<"open">; export type OpenResult = ToolResult<"open">; -export type ContextResult = ToolResult<"context">; +export type ContextResult = ToolOutput<"context">; export type CaptureRequest = ToolArgs<"capture">; export type CaptureResult = ToolResult<"capture">; export type CheckRequest = ToolArgs<"check">; @@ -64,6 +67,7 @@ export type ExportResult = ToolResult<"export">; export type ModelsRequest = ToolArgs<"models">; export type LogsRequest = ToolArgs<"logs">; export type ScreenshotResult = ToolResult<"screenshot">; +export type ScreenshotOutput = ToolOutput<"screenshot">; export type MediaProbeRequest = ToolArgs<"media_probe">; export type MediaFrameRequest = ToolArgs<"media_grab">; export type MediaFrameResult = ToolResult<"media_grab">; diff --git a/packages/dapi/src/ipc.ts b/packages/dapi/src/ipc.ts new file mode 100644 index 00000000..db0eae92 --- /dev/null +++ b/packages/dapi/src/ipc.ts @@ -0,0 +1,27 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The channel between the app's main process, which hosts the MCP server, +// and the renderer, which answers renderer tools. One call, one reply, and +// a cancel when the caller gives up. Arguments arrive validated: main parsed +// them against the catalog before forwarding. + +import type { DapiErrorCode } from "./errors"; + +export const DAPI_WIRE = { + /** main → renderer */ + CALL: "dapi:call", + /** main → renderer */ + CANCEL: "dapi:cancel", + /** renderer → main */ + REPLY: "dapi:reply", +} as const; + +export type DapiCall = { id: string; tool: string; args: unknown }; + +export type DapiCancel = { id: string }; + +export type DapiReply = + | { id: string; ok: true; data: unknown } + | { id: string; ok: false; error: { code?: DapiErrorCode; message: string } }; diff --git a/packages/dapi/src/schemas.ts b/packages/dapi/src/schemas.ts index 7557a8c7..71f8a24a 100644 --- a/packages/dapi/src/schemas.ts +++ b/packages/dapi/src/schemas.ts @@ -35,14 +35,25 @@ export const MAX_FRAMES_PER_SHEET = 12; export const Bytes = z.custom((value) => value instanceof Uint8Array, "expected bytes (a Uint8Array)"); /** - * One image: a single frame stamped with its timecode, or a contact sheet - * stamped with the span it covers (`0f-08s10f`). + * One rendered image as a handler returns it: a single frame stamped with its + * timecode, or a contact sheet stamped with the span it covers (`0f-08s10f`). */ export const TimecodedImage = z.object({ timecode: z.string(), png: Bytes, }); +/** The same image once the server has written it to disk. */ +export const ImageRef = z.object({ + timecode: z.string(), + path: z.string().describe("absolute path of the PNG"), +}); + +export const outputDirField = z + .string() + .optional() + .describe("absolute directory to write the PNGs into (default: a fresh directory under the system temp dir)"); + /** * How frames are laid out: merged into contact sheets (the default) or one * image each. `perSheet` only means something for sheets; `checkSheetOptions` diff --git a/packages/dapi/src/socket.test.ts b/packages/dapi/src/socket.test.ts new file mode 100644 index 00000000..cdba741e --- /dev/null +++ b/packages/dapi/src/socket.test.ts @@ -0,0 +1,111 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { mkdtempSync, rmSync } from "node:fs"; +import { connect, createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { LoggingMessageNotificationSchema } from "@modelcontextprotocol/sdk/types.js"; +import { tools } from "./catalog"; +import { SocketTransport } from "./socket"; + +import type { Server, Socket } from "node:net"; + +// A server over the real catalog with stub handlers, the way the app hosts +// it: one McpServer per accepted connection, MCP framed over the socket. +const dir = mkdtempSync(join(tmpdir(), "dapi-test-")); +const path = join(dir, "app.sock"); +let server: Server; + +beforeAll(async () => { + server = createServer((socket) => { + const session = new McpServer({ name: "test", version: "0.0.0" }, { capabilities: { logging: {} } }); + for (const tool of tools) { + session.registerTool( + tool.name, + { description: tool.description, inputSchema: tool.input, outputSchema: tool.output }, + async (_args, extra) => { + if (tool.name === "context") { + // A notification right before the response, in the same write burst: + // the transport must not let the response overtake its handling. + await extra.sendNotification({ method: "notifications/message", params: { level: "info", data: "about to reply" } }); + const output = { rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [] }; + return { content: [{ type: "text", text: JSON.stringify(output) }], structuredContent: output }; + } + return { isError: true, content: [{ type: "text", text: `stub has no ${tool.name}` }] }; + }, + ); + } + void session.connect(new SocketTransport(socket)); + }); + await new Promise((resolve) => server.listen(path, resolve)); +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); +}); + +async function open(): Promise<{ client: Client; socket: Socket }> { + const socket = await new Promise((resolve, reject) => { + const s = connect(path); + s.once("connect", () => resolve(s)); + s.once("error", reject); + }); + const client = new Client({ name: "test-client", version: "0.0.0" }); + await client.connect(new SocketTransport(socket)); + return { client, socket }; +} + +describe("MCP over the socket", () => { + it("lists the whole catalog with JSON Schema inputs", async () => { + const { client } = await open(); + try { + const { tools: listed } = await client.listTools(); + expect(listed.map((t) => t.name).sort()).toEqual(tools.map((t) => t.name).sort()); + const grab = listed.find((t) => t.name === "media_grab")!; + expect(grab.inputSchema.type).toBe("object"); + expect(Object.keys(grab.inputSchema.properties ?? {})).toContain("times"); + expect(grab.outputSchema?.type).toBe("object"); + } finally { + await client.close(); + } + }); + + it("returns structured content and rejects bad arguments with the field named", async () => { + const { client } = await open(); + try { + const messages: string[] = []; + client.setNotificationHandler(LoggingMessageNotificationSchema, (n) => { + messages.push(String(n.params.data)); + }); + const result = await client.callTool({ name: "context", arguments: {} }); + expect(result.structuredContent).toEqual({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [] }); + expect(messages).toEqual(["about to reply"]); + + // Bad arguments come back as a result the agent can read, not a protocol error. + const bad = await client.callTool({ name: "media_grab", arguments: { path: "/c.mp4", count: 0 } }); + expect(bad.isError).toBe(true); + expect(JSON.stringify(bad.content)).toMatch(/count/); + } finally { + await client.close(); + } + }); + + it("serves several sessions at once", async () => { + const a = await open(); + const b = await open(); + try { + const [ra, rb] = await Promise.all([a.client.ping(), b.client.ping()]); + expect(ra).toBeDefined(); + expect(rb).toBeDefined(); + } finally { + await a.client.close(); + await b.client.close(); + } + }); +}); diff --git a/packages/dapi/src/socket.ts b/packages/dapi/src/socket.ts index d38ad1c9..a46fb457 100644 --- a/packages/dapi/src/socket.ts +++ b/packages/dapi/src/socket.ts @@ -2,15 +2,94 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +// The Node side of the protocol: where the app's socket lives and how MCP +// travels over it. Its own entry point (`@diffusionstudio/dapi/socket`) so +// the renderer can import the catalog without node:os or the MCP SDK. + import { platform, tmpdir } from "node:os"; import { join } from "node:path"; +import { ReadBuffer, serializeMessage } from "@modelcontextprotocol/sdk/shared/stdio.js"; + +import type { Socket } from "node:net"; +import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js"; +import type { JSONRPCMessage } from "@modelcontextprotocol/sdk/types.js"; // One socket / named pipe per host. On macOS tmpdir is per-user; on Linux /tmp // is global but the socket file's owner-only mode 0600 keeps it isolated. -// -// Its own entry point (`@diffusionstudio/dapi/socket`) so the renderer can -// import the catalog without pulling in node:os / node:path. export const SOCKET_PATH = platform() === "win32" ? "\\\\.\\pipe\\diffusion-studio" : join(tmpdir(), "diffusion-studio.sock"); + +/** + * MCP over a local socket, framed exactly like MCP over stdio: one JSON-RPC + * message per line. The same class serves the app (one per accepted + * connection) and the CLI (one per command); a stdio proxy needs neither, + * since the bytes on the socket are already what a stdio client expects. + */ +export class SocketTransport implements Transport { + onclose?: () => void; + onerror?: (error: Error) => void; + onmessage?: (message: JSONRPCMessage) => void; + + private readonly socket: Socket; + private readonly buffer = new ReadBuffer(); + private started = false; + private draining = false; + + constructor(socket: Socket) { + this.socket = socket; + } + + async start(): Promise { + if (this.started) throw new Error("SocketTransport already started"); + this.started = true; + this.socket.on("data", (chunk: Buffer) => { + this.buffer.append(chunk); + void this.drain(); + }); + this.socket.on("error", (error) => this.onerror?.(error)); + this.socket.on("close", () => this.onclose?.()); + } + + send(message: JSONRPCMessage): Promise { + return new Promise((resolve, reject) => { + if (this.socket.destroyed) { + reject(new Error("Socket is closed")); + return; + } + this.socket.write(serializeMessage(message), (error) => (error ? reject(error) : resolve())); + }); + } + + async close(): Promise { + this.socket.end(); + this.socket.destroy(); + } + + // Messages are handed over one per turn of the event loop. The SDK runs + // notification handlers on a microtask, so a notification and the response + // that follows it in the same chunk must not be dispatched back to back: + // the response would settle the request before the notification's handler + // ran. A drain in progress picks up chunks appended meanwhile. + private async drain(): Promise { + if (this.draining) return; + this.draining = true; + try { + for (;;) { + let message: JSONRPCMessage | null; + try { + message = this.buffer.readMessage(); + } catch (error) { + this.onerror?.(error as Error); + return; + } + if (!message) return; + this.onmessage?.(message); + await new Promise((resolve) => setImmediate(resolve)); + } + } finally { + this.draining = false; + } + } +} diff --git a/packages/dapi/src/tool.ts b/packages/dapi/src/tool.ts index 7c4f1818..6e2febdf 100644 --- a/packages/dapi/src/tool.ts +++ b/packages/dapi/src/tool.ts @@ -14,7 +14,8 @@ export type RunsIn = "renderer" | "main"; export interface Tool< Name extends string = string, Input extends z.ZodObject = z.ZodObject, - Output extends z.ZodType = z.ZodType, + Output extends z.ZodObject = z.ZodObject, + Result extends z.ZodType = Output, > { /** MCP tool name: `[a-z0-9_]`, unique across the catalog. */ readonly name: Name; @@ -24,13 +25,26 @@ export interface Tool< readonly description: string; /** Always an object: MCP tool arguments are a JSON object by definition. */ readonly input: Input; + /** What the caller receives: a JSON object, the tool's structured content. */ readonly output: Output; + /** + * What the handler returns, when that is not the output: image tools hand + * back bytes, and the server presents them as files and inline images. Same + * as `output` when omitted. + */ + readonly result?: Result; readonly runsIn: RunsIn; } +/** A tool with its specifics erased, for code that iterates the catalog. */ +export type GenericTool = Tool; + /** Identity with inference: keeps the literal name and the exact schema types. */ -export function defineTool( - tool: Tool, -): Tool { +export function defineTool< + const Name extends string, + Input extends z.ZodObject, + Output extends z.ZodObject, + Result extends z.ZodType = Output, +>(tool: Tool): Tool { return tool; } diff --git a/packages/dapi/src/tools/capture.ts b/packages/dapi/src/tools/capture.ts index 56328380..8ceca5c3 100644 --- a/packages/dapi/src/tools/capture.ts +++ b/packages/dapi/src/tools/capture.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { defineTool } from "../tool"; -import { checkSheetOptions, SceneId, sheetFields, TimecodedImage } from "../schemas"; +import { checkSheetOptions, ImageRef, outputDirField, SceneId, sheetFields, TimecodedImage } from "../schemas"; export const capture = defineTool({ name: "capture", @@ -19,8 +19,10 @@ export const capture = defineTool({ .optional() .describe("positions to capture as frame numbers relative to the export's first frame, the workarea's start (default: [0])"), ...sheetFields, + output: outputDirField, }) .superRefine(checkSheetOptions), - output: z.array(TimecodedImage), + output: z.object({ images: z.array(ImageRef) }), + result: z.array(TimecodedImage), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.ts index 9c30c007..7bb408fc 100644 --- a/packages/dapi/src/tools/context.ts +++ b/packages/dapi/src/tools/context.ts @@ -30,24 +30,18 @@ export const context = defineTool({ description: "Report the current app context: the application root folder (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and where its `generate.*` declarations stand. Poll it to wait for generations without blocking.", input: z.object({}), - output: z.union([ - z.object({ - rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"), - projectDir: z.null(), - }), - z.object({ - rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"), - projectDir: z.string().describe("absolute path of the open project"), - currentTime: z - .number() - .nullable() - .describe("playhead in seconds, the unit the source places clips in; null when no scene is active"), - fontFamilies: z - .array(z.string()) - .describe("families registered in the world drawing the project; the editor default is always among them"), - generations: z.array(GenerationRow), - }), - ]), + output: z.object({ + rootDir: z.string().nullable().describe("folder projects live under; null until one has been chosen"), + projectDir: z.string().nullable().describe("absolute path of the open project; null when none is open"), + currentTime: z + .number() + .nullable() + .describe("playhead in seconds, the unit the source places clips in; null when no scene is active or no project is open"), + fontFamilies: z + .array(z.string()) + .describe("families registered in the world drawing the project; the editor default is always among them"), + generations: z.array(GenerationRow), + }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/fonts.ts b/packages/dapi/src/tools/fonts.ts index 4936dbfe..e6bdd994 100644 --- a/packages/dapi/src/tools/fonts.ts +++ b/packages/dapi/src/tools/fonts.ts @@ -29,6 +29,6 @@ export const fonts = defineTool({ style: FontStyle.optional().describe("filter to variants with the given style"), limit: z.int().min(1).optional().describe("return at most this many families"), }), - output: z.array(FontFamily), + output: z.object({ families: z.array(FontFamily) }), runsIn: "main", }); diff --git a/packages/dapi/src/tools/logs.ts b/packages/dapi/src/tools/logs.ts index d959265b..6e63ed8a 100644 --- a/packages/dapi/src/tools/logs.ts +++ b/packages/dapi/src/tools/logs.ts @@ -15,6 +15,6 @@ export const logs = defineTool({ tail: z.int().min(1).optional().describe("return only the last n entries"), level: LogLevel.optional().describe("minimum level to include"), }), - output: z.array(LogEntry), + output: z.object({ entries: z.array(LogEntry) }), runsIn: "main", }); diff --git a/packages/dapi/src/tools/media-filmstrip.ts b/packages/dapi/src/tools/media-filmstrip.ts index dd9cdd1c..20321daf 100644 --- a/packages/dapi/src/tools/media-filmstrip.ts +++ b/packages/dapi/src/tools/media-filmstrip.ts @@ -9,6 +9,7 @@ import { AssetPath, Bytes, checkWindow, windowFields } from "../schemas"; /** The window and scale that filmstrip and waveform share. */ export const previewFields = { ...windowFields, + output: z.string().optional().describe("absolute path to write the PNG to (default: a fresh file under the system temp dir)"), scale: z .number() .positive() @@ -22,6 +23,7 @@ export const mediaFilmstrip = defineTool({ description: "Render a grid of thumbnails sampled across the timeline to a PNG (local render, no credits), each row stamped with an HH:MM:SS:FF ruler. A fast, token-efficient video track preview; narrow the window to zoom into a region of interest. Video only (use media_waveform for audio).", input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), - output: z.looseObject({ png: Bytes }), + output: z.looseObject({ path: z.string().describe("absolute path of the PNG") }), + result: z.looseObject({ png: Bytes }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/media-grab.ts b/packages/dapi/src/tools/media-grab.ts index 16df6aa2..5dcbb544 100644 --- a/packages/dapi/src/tools/media-grab.ts +++ b/packages/dapi/src/tools/media-grab.ts @@ -4,7 +4,7 @@ import { z } from "zod"; import { defineTool } from "../tool"; -import { AssetPath, checkSheetOptions, checkWindow, sheetFields, TimecodedImage, windowFields } from "../schemas"; +import { AssetPath, checkSheetOptions, checkWindow, ImageRef, outputDirField, sheetFields, TimecodedImage, windowFields } from "../schemas"; import { Time } from "../time"; export const FrameQuality = z.enum(["small", "medium", "large", "fullres"]); @@ -46,6 +46,7 @@ export const mediaGrab = defineTool({ .boolean() .optional() .describe(`lift the ${FRAME_CAP}-frame safety cap (grabbing many frames is slow and token-heavy)`), + output: outputDirField, }) .superRefine((value, ctx) => { if (value.times !== undefined && value.count !== undefined) { @@ -69,6 +70,7 @@ export const mediaGrab = defineTool({ } checkSheetOptions(value, ctx); }), - output: z.array(TimecodedImage), + output: z.object({ images: z.array(ImageRef) }), + result: z.array(TimecodedImage), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/media-waveform.ts b/packages/dapi/src/tools/media-waveform.ts index d0691d5d..dd9f98ff 100644 --- a/packages/dapi/src/tools/media-waveform.ts +++ b/packages/dapi/src/tools/media-waveform.ts @@ -7,6 +7,8 @@ import { defineTool } from "../tool"; import { AssetPath, Bytes, checkWindow } from "../schemas"; import { previewFields } from "./media-filmstrip"; +const Silences = z.array(z.object({ start: z.number(), end: z.number() })).describe("seconds"); + export const mediaWaveform = defineTool({ name: "media_waveform", title: "Waveform preview", @@ -14,8 +16,9 @@ export const mediaWaveform = defineTool({ "Render the audio track of a video or audio file as a waveform PNG (local render, no credits) with a timestamp ruler: loudness over time, with silent stretches highlighted in red. A fast, token-efficient audio track preview; the silent spans are also returned as second ranges.", input: z.object({ path: AssetPath, ...previewFields }).superRefine(checkWindow), output: z.looseObject({ - png: Bytes, - silences: z.array(z.object({ start: z.number(), end: z.number() })).describe("seconds"), + path: z.string().describe("absolute path of the PNG"), + silences: Silences, }), + result: z.looseObject({ png: Bytes, silences: Silences }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/models.ts b/packages/dapi/src/tools/models.ts index f4dade8e..ca2cc8ce 100644 --- a/packages/dapi/src/tools/models.ts +++ b/packages/dapi/src/tools/models.ts @@ -24,6 +24,6 @@ export const models = defineTool({ input: z.object({ type: ModelType.optional().describe("filter to one kind of model (default: all three)"), }), - output: z.array(ModelInfo), + output: z.object({ models: z.array(ModelInfo) }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/screenshot.ts b/packages/dapi/src/tools/screenshot.ts index 9f54bd58..7cc1fa33 100644 --- a/packages/dapi/src/tools/screenshot.ts +++ b/packages/dapi/src/tools/screenshot.ts @@ -4,18 +4,19 @@ import { z } from "zod"; import { defineTool } from "../tool"; -import { Bytes } from "../schemas"; +import { Bytes, outputDirField } from "../schemas"; export const screenshot = defineTool({ name: "screenshot", title: "Window screenshot", description: "Capture the entire application window as a PNG — the full UI as the user sees it (panels, timeline, asset library, canvas viewport), at the window's current size. The tool for checking what the app itself looks like; to render a node or scene cleanly for composition checks use capture instead.", - input: z.object({}), + input: z.object({ output: outputDirField }), output: z.object({ - png: Bytes, + path: z.string().describe("absolute path of the PNG"), width: z.number(), height: z.number(), }), + result: z.object({ png: Bytes, width: z.number(), height: z.number() }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.ts index 31aa8cb7..0879c0a6 100644 --- a/packages/dapi/src/tools/tools.test.ts +++ b/packages/dapi/src/tools/tools.test.ts @@ -94,8 +94,10 @@ describe("logs and export", () => { }); describe("context", () => { - it("accepts both the closed and the open report", () => { - expect(context.output.safeParse({ rootDir: "/p", projectDir: null }).success).toBe(true); + it("reports the same shape with and without an open project", () => { + expect( + context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [] }).success, + ).toBe(true); expect( context.output.safeParse({ rootDir: "/p", @@ -108,3 +110,11 @@ describe("context", () => { expect(context.output.safeParse({ rootDir: "/p", projectDir: "/p/a" }).success).toBe(false); }); }); + +describe("image tools", () => { + it("present bytes as paths: the result carries png, the output a path", () => { + expect(capture.result!.safeParse([{ timecode: "0f", png: new Uint8Array(3) }]).success).toBe(true); + expect(capture.output.safeParse({ images: [{ timecode: "0f", path: "/tmp/0f.png" }] }).success).toBe(true); + expect(capture.output.safeParse({ images: [{ timecode: "0f", png: new Uint8Array(3) }] }).success).toBe(false); + }); +}); diff --git a/packages/dapi/src/tools/voices.ts b/packages/dapi/src/tools/voices.ts index 233cc66a..76d0f328 100644 --- a/packages/dapi/src/tools/voices.ts +++ b/packages/dapi/src/tools/voices.ts @@ -16,6 +16,6 @@ export const voices = defineTool({ title: "Speech voices", description: "List the speech voices available for `generate.voice` declarations in a project module.", input: z.object({}), - output: z.array(VoiceInfo), + output: z.object({ voices: z.array(VoiceInfo) }), runsIn: "renderer", }); diff --git a/packages/dapi/src/tools/whoami.ts b/packages/dapi/src/tools/whoami.ts index f2d5cf0b..0aa6393a 100644 --- a/packages/dapi/src/tools/whoami.ts +++ b/packages/dapi/src/tools/whoami.ts @@ -10,6 +10,8 @@ export const whoami = defineTool({ title: "Signed-in account", description: "Report the authenticated account, or null if signed out.", input: z.object({}), - output: z.object({ id: z.string(), email: z.string().optional() }).nullable(), + output: z.object({ + user: z.object({ id: z.string(), email: z.string().optional() }).nullable(), + }), runsIn: "renderer", }); diff --git a/packages/dapi/src/validate.test.ts b/packages/dapi/src/validate.test.ts deleted file mode 100644 index 6a14b389..00000000 --- a/packages/dapi/src/validate.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import { describe, expect, it } from "vitest"; -import { isDapiError } from "./errors"; -import { parseToolArgs } from "./validate"; - -describe("parseToolArgs", () => { - it("returns parsed arguments with defaults applied", () => { - expect(parseToolArgs("capture", { id: "intro" })).toEqual({ id: "intro", combine: true }); - }); - - it("throws an invalid-input DapiError that names every bad field on one line", () => { - try { - parseToolArgs("media_grab", { path: "/c.mp4", times: ["abc"], count: 0 }); - } catch (e) { - expect(isDapiError(e) && e.code).toBe("invalid-input"); - const message = (e as Error).message; - expect(message).toMatch(/^Invalid arguments for media_grab: /); - expect(message).toMatch(/times\.0: expected a time/); - expect(message).toMatch(/count: /); - expect(message).not.toContain("\n"); - return; - } - throw new Error("expected a throw"); - }); -}); diff --git a/packages/dapi/src/validate.ts b/packages/dapi/src/validate.ts deleted file mode 100644 index 89c5e776..00000000 --- a/packages/dapi/src/validate.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -import type { z } from "zod"; -import { toolByName } from "./catalog"; -import type { ToolArgs, ToolName } from "./catalog"; -import { DapiError } from "./errors"; - -/** - * Validates a tool's raw arguments against its schema. Every issue is - * reported on one line, keyed by the field it points at, so an agent can - * correct the call from the message alone. - */ -export function parseToolArgs(name: N, raw: unknown): ToolArgs { - const result = toolByName(name).input.safeParse(raw); - if (result.success) return result.data as ToolArgs; - throw new DapiError("invalid-input", `Invalid arguments for ${name}: ${formatIssues(result.error.issues)}`, { - cause: result.error, - }); -} - -function formatIssues(issues: readonly z.core.$ZodIssue[]): string { - return issues.map((issue) => `${issue.path.map(String).join(".") || "input"}: ${issue.message}`).join("; "); -} From a7039fe8b2d39cdb4432934b9c3c944f86ca3307 Mon Sep 17 00:00:00 2001 From: konstantin-paulus Date: Tue, 8 Sep 2026 12:17:03 +0200 Subject: [PATCH 04/13] Enhance MCP server integration and add help functionality - Added `zod` as a dependency in package-lock.json and apps/cli/package.json for input validation. - Introduced new help.ts module to provide descriptions and validation for CLI commands, ensuring consistency with the app's tool catalog. - Refactored cli-client.ts to improve timeout handling for tool calls, enhancing error management. - Created mcp-proxy.ts to facilitate communication between agents and the app's MCP server. - Updated index.ts to streamline command execution and improve error handling. - Implemented mcp-config.ts and mcp-install.ts for managing MCP server configurations across different agents. - Enhanced desktop application to support MCP registration from the menu, improving user experience. --- apps/cli/package.json | 3 +- apps/cli/src/cli-client.ts | 83 ++- apps/cli/src/help.ts | 44 ++ apps/cli/src/index.ts | 827 +++++++----------------- apps/cli/src/mcp-proxy.ts | 40 ++ apps/desktop/scripts/stage-cli.mjs | 3 +- apps/desktop/src/dapi/http.test.ts | 99 +++ apps/desktop/src/dapi/http.ts | 133 ++++ apps/desktop/src/dapi/knowledge.test.ts | 110 ++++ apps/desktop/src/dapi/knowledge.ts | 139 ++++ apps/desktop/src/dapi/server.ts | 68 +- apps/desktop/src/main-channels.ts | 17 +- apps/desktop/src/main.ts | 19 +- apps/desktop/src/mcp-config.test.ts | 111 ++++ apps/desktop/src/mcp-config.ts | 161 +++++ apps/desktop/src/mcp-install.ts | 131 ++++ apps/desktop/src/menu.ts | 25 + apps/desktop/src/projects.ts | 116 ++-- apps/desktop/src/skills-install.ts | 191 ------ apps/web/src/pages/onboarding.tsx | 163 +++-- docs/mcp-server.md | 11 +- package-lock.json | 3 +- packages/dapi/src/socket.ts | 15 +- reference/README.md | 2 +- reference/capture.md | 4 +- reference/fetch.md | 4 +- reference/fonts.md | 15 +- reference/jsx/module.md | 5 +- reference/logs.md | 17 +- reference/media/grab.md | 4 +- reference/models.md | 16 +- reference/voices.md | 4 +- reference/whoami.md | 4 +- 33 files changed, 1559 insertions(+), 1028 deletions(-) create mode 100644 apps/cli/src/help.ts create mode 100644 apps/cli/src/mcp-proxy.ts create mode 100644 apps/desktop/src/dapi/http.test.ts create mode 100644 apps/desktop/src/dapi/http.ts create mode 100644 apps/desktop/src/dapi/knowledge.test.ts create mode 100644 apps/desktop/src/dapi/knowledge.ts create mode 100644 apps/desktop/src/mcp-config.test.ts create mode 100644 apps/desktop/src/mcp-config.ts create mode 100644 apps/desktop/src/mcp-install.ts delete mode 100644 apps/desktop/src/skills-install.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index d708d36a..3541cf91 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -19,7 +19,8 @@ "@modelcontextprotocol/sdk": "^1.30.0", "babel-preset-solid": "^1.9.12", "commander": "^14.0.3", - "esbuild": "^0.28.1" + "esbuild": "^0.28.1", + "zod": "^4.4.3" }, "devDependencies": { "@types/babel__core": "^7.20.5", diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.ts index bd3189ff..45e54a1d 100644 --- a/apps/cli/src/cli-client.ts +++ b/apps/cli/src/cli-client.ts @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import { execFile } from "node:child_process"; import { connect } from "node:net"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { SOCKET_PATH, SocketTransport } from "@diffusionstudio/dapi/socket"; @@ -10,9 +11,25 @@ import { version } from "../../../package.json"; import type { Socket } from "node:net"; import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi"; +export const APP_NAME = "Diffusion Studio"; + const DEFAULT_TIMEOUT_MS = 60000; -export const GENERATE_TIMEOUT_MS = 600000; -export const EXPORT_TIMEOUT_MS = 3600000; +const GENERATE_TIMEOUT_MS = 600000; +const EXPORT_TIMEOUT_MS = 3600000; + +// Long-running tools (renders, AI generation, downloads) override the +// default 60s. Keyed by name so `call` and the wrappers agree. +const TIMEOUTS: Record = { + export: EXPORT_TIMEOUT_MS, + capture: GENERATE_TIMEOUT_MS, + media_transcribe: GENERATE_TIMEOUT_MS, + media_listen: GENERATE_TIMEOUT_MS, + fetch: GENERATE_TIMEOUT_MS, +}; + +export function timeoutFor(tool: string): number { + return TIMEOUTS[tool] ?? DEFAULT_TIMEOUT_MS; +} export type CallOptions = { timeoutMs?: number }; @@ -22,16 +39,12 @@ export type CallOptions = { timeoutMs?: number }; * output its structured content. One session per call; a command makes one * or two, and the process exits when it settles. */ -export async function call( - name: N, - input: ToolInput, - options: CallOptions = {}, -): Promise> { +export async function call(name: N, input: ToolInput, options: CallOptions = {}): Promise> { return withClient(async (client) => { const result = await client.callTool( { name, arguments: input as Record }, undefined, - { timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS }, + { timeout: options.timeoutMs ?? timeoutFor(name) }, ); if (result.isError) { const text = (result.content as Array<{ type: string; text?: string }>) @@ -63,8 +76,8 @@ async function withClient(fn: (client: Client) => Promise): Promise { } // Connecting is where "the app is not running" shows up, as ENOENT (no -// socket file) or ECONNREFUSED (a stale one); see `errnoCode`. -function openSocket(): Promise { +// socket file) or ECONNREFUSED (a stale one); see `isAppDown`. +export function openSocket(): Promise { return new Promise((resolve, reject) => { const socket = connect(SOCKET_PATH); socket.once("connect", () => { @@ -80,9 +93,48 @@ export function errnoCode(e: unknown): string | undefined { return (e as NodeJS.ErrnoException | undefined)?.code; } -// Bridges the cold-start gap after launching the app: the socket appears -// once main is ready, and `ping` proves the server answers. The retry loop -// only handles the brief window before the socket binds. +export function isAppDown(e: unknown): boolean { + const code = errnoCode(e); + return code === "ENOENT" || code === "ECONNREFUSED"; +} + +/** + * Launches the app, or surfaces the running instance: `open -a` on a running + * app only activates it, so this is safe to always run. macOS only; elsewhere + * it resolves false and the caller falls through to the socket. + */ +export function launchApp(background: boolean): Promise { + if (process.platform !== "darwin") return Promise.resolve(false); + const args = background ? ["-g", "-a", APP_NAME, "--args", "--hidden"] : ["-a", APP_NAME]; + return new Promise((res) => execFile("open", args, (err) => res(!err))); +} + +/** + * Bridges the cold-start gap after launching the app: the socket appears once + * main is ready. Retries only while the app looks down; any other error is + * the caller's. + */ +export async function connectWithRetry(timeoutMs = 30000): Promise { + const start = Date.now(); + let lastError: unknown = null; + while (Date.now() - start < timeoutMs) { + try { + return await openSocket(); + } catch (e) { + if (!isAppDown(e)) throw e; + lastError = e; + await new Promise((r) => setTimeout(r, 200)); + } + } + throw timedOut(timeoutMs, lastError); +} + +function timedOut(timeoutMs: number, lastError: unknown): Error { + const detail = lastError instanceof Error ? ` (${lastError.message})` : ""; + return new Error(`${APP_NAME} did not answer within ${Math.round(timeoutMs / 1000)}s of launching${detail}`); +} + +/** Like `connectWithRetry`, but also proves the server answers: a cold app binds the socket before its session is ready. */ export async function waitForApp(timeoutMs = 30000): Promise { const start = Date.now(); let lastError: unknown = null; @@ -91,11 +143,10 @@ export async function waitForApp(timeoutMs = 30000): Promise { await ping(); return; } catch (e) { + if (!isAppDown(e)) throw e; lastError = e; - const code = errnoCode(e); - if (code !== "ENOENT" && code !== "ECONNREFUSED") throw e; await new Promise((r) => setTimeout(r, 200)); } } - throw lastError instanceof Error ? lastError : new Error("Timed out waiting for the app to start"); + throw timedOut(timeoutMs, lastError); } diff --git a/apps/cli/src/help.ts b/apps/cli/src/help.ts new file mode 100644 index 00000000..e902d2f9 --- /dev/null +++ b/apps/cli/src/help.ts @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The wrappers' help text and validation come from the catalog, so +// `dapi --help` and the app's `tools/list` say the same thing because +// they are the same string, and a bad argument fails here with the message +// the app would have sent back. + +import { z } from "zod"; +import { toolByName } from "@diffusionstudio/dapi"; + +import type { GenericTool, ToolName } from "@diffusionstudio/dapi"; + +/** The tool's description, verbatim. */ +export function describe(name: ToolName): string { + return toolByName(name).description; +} + +/** + * An input field's description, for the option that maps onto it. The + * fallback covers fields the catalog leaves undescribed and options whose + * meaning is the CLI's own (`--separate` inverts `combine`). + */ +export function field(name: ToolName, key: string, fallback?: string): string { + const tool: GenericTool = toolByName(name); + const schema = tool.input.shape[key]; + if (schema === undefined && fallback === undefined) { + throw new Error(`tool ${name} has no input field "${key}"`); + } + return schema?.description ?? fallback ?? ""; +} + +/** + * Checks the input against the catalog before it leaves the process, so a + * bad argument fails here with the message the app would have sent back. + * Prints the issues and exits on failure. + */ +export function validate(name: ToolName, input: Record): void { + const result = toolByName(name).input.safeParse(input); + if (result.success) return; + console.error(z.prettifyError(result.error)); + process.exit(1); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 4058434e..2216fcdb 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -3,206 +3,77 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { execFile } from "node:child_process"; import { existsSync } from "node:fs"; import { isAbsolute, resolve } from "node:path"; import { Command } from "commander"; import { version } from "../../../package.json"; -import { parseTime, TIME_FPS } from "@diffusionstudio/jsx"; -import { call, errnoCode, EXPORT_TIMEOUT_MS, GENERATE_TIMEOUT_MS, ping, waitForApp } from "./cli-client"; -import { ISSUE_LOG_TAIL, MAX_FRAMES_PER_SHEET } from "@diffusionstudio/dapi"; -import type { FrameQuality, ImageRef, LogEntry, LogLevel } from "@diffusionstudio/dapi"; +import { TIME_FPS } from "@diffusionstudio/jsx"; +import { ISSUE_LOG_TAIL, NonNegativeTime } from "@diffusionstudio/dapi"; +import { MCP_URL } from "@diffusionstudio/dapi/socket"; +import { APP_NAME, call, isAppDown, launchApp, ping, waitForApp } from "./cli-client"; +import { describe, field, validate } from "./help"; +import { runProxy } from "./mcp-proxy"; -// Long-running commands (renders, AI generation) override the default 60s. -const GENERATE = { timeoutMs: GENERATE_TIMEOUT_MS }; -const EXPORT = { timeoutMs: EXPORT_TIMEOUT_MS }; +import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi"; -const APP_NAME = "Diffusion Studio"; - -function handleSocketError(e: unknown): never { - const code = errnoCode(e); - if (code === "ENOENT" || code === "ECONNREFUSED") { - console.error(`${APP_NAME} is not running. Launch the app first, then retry.`); - } else { - console.error((e as Error).message); - } +function fail(message: string): never { + console.error(message); process.exit(1); } -const FRAME_QUALITIES: FrameQuality[] = ["small", "medium", "large", "fullres"]; - -// Guardrail against accidentally decoding a huge number of frames; --uncapped lifts it. -const FRAME_CAP = 100; - -type MediaFrameOptions = { - time?: string[]; - count?: string; - start?: string; - end?: string; - quality?: string; - uncapped?: boolean; - output?: string; - auto?: boolean; - separate?: boolean; - perSheet?: string; -}; - -async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise { - if (opts.time !== undefined && opts.count !== undefined) { - console.error("Pass either --time or --count, not both."); - process.exit(1); - } - if (opts.auto && opts.time !== undefined) { - console.error("--auto picks its own timestamps; it cannot be combined with --time."); - process.exit(1); - } - - let times: number[] | undefined; - if (opts.time !== undefined) { - times = opts.time.map((t) => parseTimeArg(t, "--time", true)); - } - - let count: number | undefined; - if (opts.count !== undefined) { - count = Number(opts.count); - if (!Number.isInteger(count) || count < 1) { - console.error(`--count must be a positive integer (got "${opts.count}")`); - process.exit(1); - } - } - - const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined; - const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined; - if (start !== undefined && end !== undefined && start >= end) { - console.error(`--start (${start}s) must be less than --end (${end}s).`); - process.exit(1); - } - if ((start !== undefined || end !== undefined) && count === undefined && !opts.auto) { - console.error("--start and --end only apply together with --count or --auto."); - process.exit(1); - } - - const requested = count ?? times?.length ?? 1; - if (!opts.uncapped && requested > FRAME_CAP) { - console.error(`Grabbing ${requested} frames exceeds the ${FRAME_CAP}-frame cap; pass --uncapped to override.`); - process.exit(1); - } - - let quality: FrameQuality | undefined; - if (opts.quality !== undefined) { - if (!FRAME_QUALITIES.includes(opts.quality as FrameQuality)) { - console.error(`--quality must be one of ${FRAME_QUALITIES.join(", ")} (got "${opts.quality}")`); - process.exit(1); - } - quality = opts.quality as FrameQuality; - } - - const perSheet = parsePerSheet(opts.perSheet, opts.separate); - const target = resolveAssetRef(ref); - try { - const { images } = await call("media_grab", { - ...target, - times, - count, - start, - end, - quality, - auto: opts.auto, - combine: !opts.separate, - perSheet, - uncapped: opts.uncapped, - output: resolveOutput(opts.output), - }); - printImages(images); - } catch (e) { - handleSocketError(e); - } +function handleSocketError(e: unknown): never { + if (isAppDown(e)) fail(`${APP_NAME} is not running. Launch the app first, then retry.`); + fail((e as Error).message); } /** - * A local file (or frames folder) that exists is sent as its absolute path; - * anything else — a URL, or a library path (`b-roll/clip.mp4`) — is passed - * through for the app to resolve. Library paths need an open project. + * One wrapper's whole job: check the input against the catalog, call the + * tool, print the result the way the app returns it — its structured content, + * as one JSON object, the same thing an agent receives. Everything the app + * validates is sent as typed (strings stay strings: times like "45f" are + * parsed by the schema on both sides). */ -function resolveAssetRef(ref: string): { path: string } { - const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref); - if (existsSync(absPath)) return { path: absPath }; - if (isAbsolute(ref)) { - console.error(`File not found: ${absPath}`); - process.exit(1); - } - return { path: ref }; -} - -async function mediaProbe(ref: string): Promise { - const target = resolveAssetRef(ref); - const stop = startSpinner("Probing asset"); - try { - const result = await call("media_probe", target); - stop(); - console.log(JSON.stringify(result)); - } catch (e) { - stop(); - handleSocketError(e); - } +async function run(name: N, input: ToolInput, spinner?: string): Promise { + await invoke(name, input, spinner); } -async function mediaTranscribe(ref: string): Promise { - const target = resolveAssetRef(ref); - const stop = startSpinner("Transcribing asset"); +/** `run`, handing the result back for the one command that inspects it (`check`, for its exit code). */ +async function invoke(name: N, input: ToolInput, spinner?: string): Promise> { + validate(name, input as Record); + const stop = spinner ? startSpinner(spinner) : () => {}; try { - const result = await call("media_transcribe", target, GENERATE); + const output = await call(name, input); stop(); - console.log(JSON.stringify(result)); + console.log(JSON.stringify(output)); + return output; } catch (e) { stop(); handleSocketError(e); } } -type MediaListenOptions = { prompt?: string; start?: string; end?: string; keepVideo?: boolean }; - -async function mediaListen(ref: string, opts: MediaListenOptions): Promise { - const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined; - const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined; - if (start !== undefined && end !== undefined && start >= end) { - console.error(`--start (${start}s) must be less than --end (${end}s).`); - process.exit(1); - } +// Option parsers. Numbers are converted so the schema can check them as +// numbers; an empty or non-numeric string becomes NaN, which the schema +// rejects with its own message. +const numeric = (value: string): number => (value.trim() === "" ? NaN : Number(value)); - const target = resolveAssetRef(ref); - const stop = startSpinner("Analyzing asset"); - try { - const result = await call( - "media_listen", - { ...target, prompt: opts.prompt, start, end, stripVideo: !opts.keepVideo }, - GENERATE, - ); - stop(); - console.log(JSON.stringify(result)); - } catch (e) { - stop(); - handleSocketError(e); - } +/** A time as the CLI takes it, in seconds, checked the way the catalog checks it. */ +function seconds(value: string, flag: string): number { + const result = NonNegativeTime.safeParse(value); + if (!result.success) fail(`${flag}: ${result.error.issues[0]?.message ?? `invalid time (got "${value}")`}`); + return result.data; } -type MediaPreviewOptions = { start?: string; end?: string; scale?: string; output?: string }; - -function parseTimeArg(value: string, flag: string, allowNegative = false): number { - const seconds = parseTime(value); - if (seconds === undefined || (!allowNegative && seconds < 0)) { - console.error( - `${flag} must be a ${allowNegative ? "" : "non-negative "}Time — seconds ("1.5"), frames ("45f"), or "MM:SS" (got "${value}")`, - ); - process.exit(1); - } - return seconds; -} - -// The app wrote the PNGs (one per frame or contact sheet, named by -// timecode); print where each landed, one line per image. -function printImages(images: ImageRef[]): void { - for (const image of images) console.log(JSON.stringify(image)); +/** + * A local file (or frames folder) that exists is sent as its absolute path; + * anything else — a URL, or a library path (`b-roll/clip.mp4`) — is passed + * through for the app to resolve. Library paths need an open project. + */ +function resolveAssetRef(ref: string): { path: string } { + const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref); + if (existsSync(absPath)) return { path: absPath }; + if (isAbsolute(ref)) fail(`File not found: ${absPath}`); + return { path: ref }; } /** An output path as the app needs it: absolute, or absent for the app's default. */ @@ -210,233 +81,10 @@ function resolveOutput(path: string | undefined): string | undefined { return path === undefined ? undefined : resolve(process.cwd(), path); } -function parsePerSheet(value: string | undefined, separate?: boolean): number | undefined { - if (value === undefined) return undefined; - if (separate) { - console.error("--per-sheet lays out contact sheets; it cannot be combined with --separate."); - process.exit(1); - } - const n = Number(value); - if (!Number.isInteger(n) || n < 1 || n > MAX_FRAMES_PER_SHEET) { - console.error(`--per-sheet must be an integer between 1 and ${MAX_FRAMES_PER_SHEET} (got "${value}")`); - process.exit(1); - } - return n; -} - -// Parse the window/scale flags shared by `filmstrip` and `waveform`. -function parsePreviewWindow(opts: MediaPreviewOptions): { start?: number; end?: number; scale?: number } { - const start = opts.start !== undefined ? parseTimeArg(opts.start, "--start") : undefined; - const end = opts.end !== undefined ? parseTimeArg(opts.end, "--end") : undefined; - if (start !== undefined && end !== undefined && start >= end) { - console.error(`--start (${start}s) must be less than --end (${end}s).`); - process.exit(1); - } - - let scale: number | undefined; - if (opts.scale !== undefined) { - scale = Number(opts.scale); - if (!Number.isFinite(scale) || scale <= 0) { - console.error(`--scale must be a positive number (got "${opts.scale}")`); - process.exit(1); - } - } - - return { start, end, scale }; -} - -async function mediaFilmstrip(ref: string, opts: MediaPreviewOptions): Promise { - const { start, end, scale } = parsePreviewWindow(opts); - const target = resolveAssetRef(ref); - const stop = startSpinner("Rendering filmstrip"); - try { - const result = await call("media_filmstrip", { ...target, start, end, scale, output: resolveOutput(opts.output) }); - stop(); - console.log(JSON.stringify(result)); - } catch (e) { - stop(); - handleSocketError(e); - } -} - -async function mediaWaveform(ref: string, opts: MediaPreviewOptions): Promise { - const { start, end, scale } = parsePreviewWindow(opts); - const target = resolveAssetRef(ref); - const stop = startSpinner("Rendering waveform"); - try { - const result = await call("media_waveform", { ...target, start, end, scale, output: resolveOutput(opts.output) }); - stop(); - console.log(JSON.stringify(result)); - } catch (e) { - stop(); - handleSocketError(e); - } -} - -type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: string }; - -async function captureNode(id: string, opts: CaptureOptions): Promise { - const times = (opts.time ?? ["0"]).map((t) => parseTimeArg(t, "--time")); - const frames = times.map((t) => Math.round(t * TIME_FPS)); - const perSheet = parsePerSheet(opts.perSheet, opts.separate); - - try { - const { images } = await call( - "capture", - { id, frames, combine: !opts.separate, perSheet, output: resolveOutput(opts.output) }, - GENERATE, - ); - printImages(images); - } catch (e) { - handleSocketError(e); - } -} - -async function exportScene(id: string, output: string | undefined): Promise { - // The app owns everything else: settings come from the project's - // package.json, the extension check and the default output path need the - // config and project folder, which live on its side of the socket. - const path = output !== undefined ? resolve(process.cwd(), output) : undefined; - const stop = startSpinner("Exporting scene"); - try { - const result = await call("export", { id, path }, EXPORT); - stop(); - console.log(JSON.stringify(result)); - } catch (e) { - stop(); - handleSocketError(e); - } -} - -async function checkNode(id: string): Promise { - try { - const result = await call("check", { id }); - console.log(JSON.stringify(result)); - // Linter convention: issues found is a different failure than "could not run". - if (result.issues.some((issue) => issue.severity === "error")) process.exitCode = 1; - } catch (e) { - handleSocketError(e); - } -} - -type OpenOptions = { background?: boolean }; - -/** `open -a` on a running app only activates it, so this is safe to always run. */ -function launchApp(background: boolean): Promise { - const args = background ? ["-g", "-a", APP_NAME, "--args", "--hidden"] : ["-a", APP_NAME]; - return new Promise((res) => execFile("open", args, (err) => res(!err))); -} - -async function openProject(path: string | undefined, opts: OpenOptions): Promise { - // Launching is macOS's job; elsewhere (and when the app is not installed, - // e.g. a dev checkout run from the terminal) fall through to the socket, - // which answers if the app is running and errors usefully if not. - const launched = process.platform === "darwin" && (await launchApp(opts.background ?? false)); - - try { - // A cold launch needs the renderer up before the app can answer; when - // nothing was launched there is nothing to wait for, so fail fast. - if (launched) await waitForApp(); - else await ping(); - - if (path !== undefined) { - const result = await call("open", { dir: resolve(path) }); - console.log(JSON.stringify(result)); - } - } catch (e) { - handleSocketError(e); - } -} - -async function context(): Promise { - try { - const result = await call("context", {}); - console.log(JSON.stringify(result)); - } catch (e) { - handleSocketError(e); - } -} - -async function whoami(): Promise { - try { - const { user } = await call("whoami", {}); - console.log(JSON.stringify(user)); - } catch (e) { - handleSocketError(e); - } -} - -const LOG_LEVELS = ["debug", "info", "warning", "error"] as const; - -type LogsOptions = { tail?: string; level?: string }; - -async function showLogs(opts: LogsOptions): Promise { - if (opts.level !== undefined && !LOG_LEVELS.includes(opts.level as LogLevel)) { - console.error(`--level must be one of ${LOG_LEVELS.join(", ")} (got "${opts.level}")`); - process.exit(1); - } - let tail: number | undefined; - if (opts.tail !== undefined) { - const n = Number(opts.tail); - if (!Number.isInteger(n) || n <= 0) { - console.error(`--tail must be a positive integer (got "${opts.tail}")`); - process.exit(1); - } - tail = n; - } - - try { - const { entries } = await call("logs", { tail, level: opts.level as LogLevel | undefined }); - for (const entry of entries) console.log(formatLogEntry(entry)); - } catch (e) { - handleSocketError(e); - } -} - -function formatLogEntry(entry: LogEntry): string { - const pad = (n: number, w = 2) => String(n).padStart(w, "0"); - const d = new Date(entry.ts); - const time = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}.${pad(d.getMilliseconds(), 3)}`; - const source = entry.source ? ` (${entry.source})` : ""; - return `${time} [${entry.level}] ${entry.message}${source}`; -} - -type ScreenshotOptions = { output?: string }; - -// `diffusion-studio_2026-07-31_08-55-12.png` -async function appScreenshot(opts: ScreenshotOptions): Promise { - try { - const result = await call("screenshot", { output: resolveOutput(opts.output) }); - console.log(JSON.stringify(result)); - } catch (e) { - handleSocketError(e); - } -} - -type IssueOptions = { body?: string; command?: string[]; logs?: string }; - -async function reportIssue(title: string, opts: IssueOptions): Promise { - let logs: number | undefined; - if (opts.logs !== undefined) { - const n = Number(opts.logs); - if (!Number.isInteger(n) || n < 0) { - console.error(`--logs must be a non-negative integer (got "${opts.logs}")`); - process.exit(1); - } - logs = n; - } - try { - const result = await call("report", { title, body: opts.body, commands: opts.command, logs }); - console.log(JSON.stringify(result)); - } catch (e) { - handleSocketError(e); - } -} - function startSpinner(label: string): () => void { if (!process.stderr.isTTY) { process.stderr.write(`${label}…\n`); - return () => { }; + return () => {}; } const frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; const start = Date.now(); @@ -454,82 +102,29 @@ function startSpinner(label: string): () => void { }; } -async function listModels(type: string | undefined): Promise { - if (type !== undefined && type !== "image" && type !== "video" && type !== "audio") { - console.error(`[type] must be one of "image", "video", "audio" (got "${type}")`); - process.exit(1); - } - try { - const { models } = await call("models", { type: type as "image" | "video" | "audio" | undefined }); - for (const model of models) console.log(JSON.stringify(model)); - } catch (e) { - handleSocketError(e); - } -} +// --------------------------------------------------------------------------- +// Commands that are the CLI's own: launching, and the proxy. -async function listVoices(): Promise { - try { - const { voices } = await call("voices", {}); - for (const voice of voices) console.log(JSON.stringify(voice)); - } catch (e) { - handleSocketError(e); - } -} - -type ListFontsOptions = { - family?: string; - weight?: string[]; - style?: string; - limit?: string; - namesOnly?: boolean; -}; - -async function listFonts(opts: ListFontsOptions): Promise { - let style: "normal" | "italic" | undefined; - if (opts.style !== undefined) { - if (opts.style !== "normal" && opts.style !== "italic") { - console.error(`--style must be "normal" or "italic" (got "${opts.style}")`); - process.exit(1); - } - style = opts.style; - } - - let limit: number | undefined; - if (opts.limit !== undefined) { - const n = Number(opts.limit); - if (!Number.isInteger(n) || n <= 0) { - console.error(`--limit must be a positive integer (got "${opts.limit}")`); - process.exit(1); - } - limit = n; - } +type OpenOptions = { background?: boolean }; +async function openProject(path: string | undefined, opts: OpenOptions): Promise { + // Launching is macOS's job; elsewhere (and when the app is not installed, + // e.g. a dev checkout run from the terminal) fall through to the socket, + // which answers if the app is running and errors usefully if not. + const launched = await launchApp(opts.background ?? false); try { - const { families } = await call("fonts", { family: opts.family, weights: opts.weight, style, limit }); - if (opts.namesOnly) { - for (const family of families) console.log(family.family); - } else { - for (const family of families) console.log(JSON.stringify(family)); - } + // A cold launch needs the renderer up before the app can answer; when + // nothing was launched there is nothing to wait for, so fail fast. + if (launched) await waitForApp(); + else await ping(); } catch (e) { handleSocketError(e); } + if (path !== undefined) await run("open", { dir: resolve(path) }); } -type FetchCliOptions = { output?: string; format?: string; audio?: boolean }; - -// `raw` is every operand after `url` — the yt-dlp passthrough placed after `--`. -async function fetch(url: string, opts: FetchCliOptions, raw: string[]): Promise { - const stop = startSpinner("Downloading"); - try { - const { paths } = await call("fetch", { url, ...opts, raw, output: resolveOutput(opts.output) }, GENERATE); - stop(); - for (const path of paths) console.log(JSON.stringify({ path })); - } catch (e) { - stop(); - handleSocketError(e); - } -} +// --------------------------------------------------------------------------- +// The program. const program = new Command(); @@ -538,58 +133,68 @@ program .description( `The Diffusion Studio CLI: understand, generate, and edit footage. Analyze video/audio/images, generate them with AI, and compose assets. -Use for any media analysis, media generation, or video editing task. No ffmpeg needed.`, +Use for any media analysis, media generation, or video editing task. No ffmpeg needed. + +Every command wraps one tool of the running app's MCP server, which agents reach at ${MCP_URL} once the app runs (\`dapi open\`).`, ) .version(version); +program + .command("mcp") + .description( + `Serve the app's MCP server on stdio, for agents that cannot connect over HTTP (Claude Desktop). Every other agent should register the URL ${MCP_URL} instead, which the running app serves — \`dapi open\` starts it. Register this with \`claude mcp add dapi -- dapi mcp\` (or the equivalent entry in the agent's MCP config) and the agent gets every command below as a tool, with the same descriptions. Launches ${APP_NAME} in the background if it is not running (macOS).`, + ) + .action(() => runProxy().catch((e: Error) => fail(e.message))); + program .command("open") .description( - `Launch ${APP_NAME} (or surface the running instance) and, given a path, open that folder as a project.`, + `Launch ${APP_NAME} (or surface the running instance) and, given a path, open that folder as a project, creating the project files if the folder is not one yet. Prints the project's id, display name, and folder. Run this once before commands that need an open project (capture, check, export, context, and library paths in media commands).`, ) - .argument("[path]", "project folder to open or create (default: none — just launch the app)") + .argument("[path]", `${field("open", "dir")} (default: none — just launch the app)`) .option("-b, --background", "launch or keep the app in the background, without raising a window") .action((path: string | undefined, opts: OpenOptions) => openProject(path, opts)); program .command("context") .alias("ctx") - .description( - `Print the current app context: the application root folder (always reported), the folder of the project the app has open (null when none is), where its playhead sits, in seconds, the registered font families, and where its generations stand.`, - ) - .action(() => context()); + .description(describe("context")) + .action(() => run("context", {})); + +type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: number }; program .command("capture") - .description( - `Render single frames of a scene to PNGs — each frame is the frame an export of that scene would encode, drawn offscreen at the scene's own size. By default the positions are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (\`08s10f\`, zero segments dropped) and rendered as large as fits, so a few positions arrive as one high-resolution picture instead of a directory to open one by one (\`--separate\` writes a PNG per position, at 720p height). The tool for checking composition ("what plays at time T": layout, overlaps, text, timing) and for verifying frames before an export. Scenes only — a single element renders inside its scene, so capture the scene at the times it plays. For a video asset's own full-resolution pixels use \`media grab\`.`, + .description(describe("capture")) + .argument("", field("capture", "id")) + .option( + "-t, --time ", + `one or more positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`, ) - .argument("", 'scene id to capture or `file:id` when two files use the same id') - .option("-t, --time ", `one or more positions to capture, relative to the export's first frame, the workarea's start (0 = the export's frame 0) — seconds ("1.5"), frames ("45f"), or "MM:SS" (default: 0)`) - .option("-S, --separate", "write one PNG per position instead of merging them into contact sheets") - .option("--per-sheet ", "positions per contact sheet, 1-12; fewer means a larger cell each (default: as many as fit)") - .option("-o, --output ", "directory to write the PNGs into (default: a fresh dir in the system temp dir)") - .action((id: string, opts: CaptureOptions) => captureNode(id, opts)); + .option("-S, --separate", "write one PNG per position instead of merging them into contact sheets (combine: false)") + .option("--per-sheet ", field("capture", "perSheet"), numeric) + .option("-o, --output ", field("capture", "output")) + .action((id: string, opts: CaptureOptions) => { + const frames = (opts.time ?? ["0"]).map((t) => Math.round(seconds(t, "--time") * TIME_FPS)); + return run("capture", { id, frames, combine: !opts.separate, perSheet: opts.perSheet, output: resolveOutput(opts.output) }); + }); program .command("export") - .description( - `Encode a scene to a video file — the same render the app's export runs, covering the scene's workarea. Settings come from the scene's \`diffusion.export.\` entry in the project's package.json (the entry the app's export panel writes); a scene without one exports with the defaults (1080p H.264 MP4, AAC audio). The [output] extension picks the container, overriding the configured format. Prints one JSON object with the written path and the settings used. One export runs at a time; progress shows in the app.`, - ) - .argument("", 'scene id to export or `file:id` when two files use the same id') - .argument( - "[output]", - 'output file path, ffmpeg-style; its extension picks the container (default: "exports/." in the project folder)', - ) - .action((id: string, output: string | undefined) => exportScene(id, output)); + .description(describe("export")) + .argument("", field("export", "id")) + .argument("[output]", field("export", "path")) + .action((id: string, output: string | undefined) => run("export", { id, path: resolveOutput(output) }, "Exporting scene")); program .command("check") - .description( - `Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load or generate — plus subtree stats (node count by kind, nesting depth, played duration). Prints one JSON object; times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock \`capture --time\` uses. Exits 1 when an error-severity issue is found. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with \`capture\`.`, - ) - .argument("", 'node id to check or `file:id` when two files use the same id') - .action((id: string) => checkNode(id)); + .description(`${describe("check")} Exits 1 when an error-severity issue is found.`) + .argument("", field("check", "id")) + .action(async (id: string) => { + const { issues } = await invoke("check", { id }); + // Linter convention: issues found is a different failure than "could not run". + if (issues.some((issue) => issue.severity === "error")) process.exitCode = 1; + }); const media = program .command("media") @@ -600,148 +205,186 @@ const media = program media .command("probe") - .description( - `Read the container and per-track technical metadata of a media file (local read, no credits): container format, duration, tags, and each track's codec params, without decoding. Commonly useful for a quick technical read, e.g. checking codec compatibility or duration before cutting. Packet stats (fps, bitrate) are estimated from a leading sample; images and transcripts report file-level info only.`, - ) - .argument("", "local file path") - .action((ref: string) => mediaProbe(ref)); + .description(describe("media_probe")) + .argument("", field("media_probe", "path")) + .action((ref: string) => run("media_probe", resolveAssetRef(ref), "Probing asset")); media .command("transcribe") - .description( - `Transcribe the speech in a video or audio file and print the timed transcript, with word-level start/end times in seconds. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).`, - ) - .argument("", "local video or audio file path") - .action((ref: string) => mediaTranscribe(ref)); + .description(describe("media_transcribe")) + .argument("", field("media_transcribe", "path")) + .action((ref: string) => run("media_transcribe", resolveAssetRef(ref), "Transcribing asset")); + +type GrabOptions = { + time?: string[]; + count?: number; + start?: string; + end?: string; + quality?: string; + uncapped?: boolean; + output?: string; + auto?: boolean; + separate?: boolean; + perSheet?: number; +}; media .command("grab") .alias("sample") - .description( - `Decode frames of a video file and write them as PNGs (local render, no credits). By default the frames are merged into contact sheets: up to 12 per image, each cell labelled with its timecode (\`08s10f\`, zero segments dropped) and drawn as large as fits, so a handful of frames arrives as one high-resolution picture instead of a directory to open one by one (\`--separate\` writes a PNG per frame). Grabs the asset's own pixels, unlike \`capture\` which renders the composited node. The recommended tool for understanding a video at the frame level; past ~12 frames prefer \`media filmstrip\`.`, - ) - .argument("", "local video file path to grab frames from") - .option("-t, --time ", `one or more timestamps to grab — seconds ("1.5"), frames ("45f"), or "MM:SS"; negatives count back from the end, so -1 is one second before the end and -1f one frame before it (default: 0)`) - .option("-c, --count ", "instead of --time, grab this many frames evenly spaced across the clip (or across the --start/--end window)") - .option("-a, --auto", "scan the clip at 2fps and keep a frame each time the footage settles into a new visual state (transitions are waited out, so picks stay sharp); returns at most --count frames (default cap: 30), static footage like screen recordings returns far fewer; requires WebGPU") - .option("-s, --start