diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ef99c1..f50d735 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,10 +10,25 @@ on: description: 'Skip npm publish (for testing)' type: boolean default: true + skip_gpu: + description: 'Skip GPU tests (alpha cuts — CPU/Metal matrix still gates)' + type: boolean + default: false + dl_source_run_id: + description: >- + Resume the DL chain for a version whose archives ALREADY uploaded: the + id of the run whose build-dl-flavor succeeded and whose chain failed + after it. Runs only the L4 DL gate and the signed-manifest publish, + against that run's archives and manifest. The pack is deterministic + and the channel paths are write-once, so this is the one way to + finish such a version. Empty for a normal release. + type: string + default: '' jobs: build-and-test: name: Build & Test ${{ matrix.package }} + if: inputs.dl_source_run_id == '' runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -210,7 +225,9 @@ jobs: uses: actions/cache@v4 with: path: models/ - key: test-models-v1 + # v2: + SmolVLM-256M pair (multimodal CI tier). actions/cache never + # re-saves on a hit — bump the key when matrix.json gains files. + key: test-models-v2 - name: Download test models if: matrix.cross_compile != true @@ -327,6 +344,7 @@ jobs: gpu-tests: name: GPU Tests needs: build-and-test + if: github.event_name == 'push' || inputs.skip_gpu != true uses: ./.github/workflows/gpu-test.yml secrets: inherit permissions: @@ -355,6 +373,7 @@ jobs: # ubuntu-24.04 runners building inside an ubuntu:22.04 container, # so the glibc floor survives the runner-image move. runs-on: ubuntu-22.04 + if: inputs.dl_source_run_id == '' outputs: uploaded: ${{ steps.upload.outputs.uploaded }} version: ${{ steps.pack.outputs.version }} @@ -562,10 +581,71 @@ jobs: # assertion → the full model-matrix inference suites. Strictly additive # to the rig's two existing consumers: new script, new JOB_NAME, zero # edits to run-gpu-tests.sh or existing job definitions. + # Resume: the archives and manifest of an EARLIER run of this same version, + # whose DL chain failed after its upload. The write-once paths refuse a second + # upload and the pack is deterministic, so the uploaded bytes ARE the pack; + # what is missing is the L4 gate and the signing act, which run below exactly + # as in a normal release, fed from this job instead of build-dl-flavor. The + # manifest artifact carries the file names and hashes; the URLs derive from + # the version, which must be the version this checkout declares. + dl-source: + name: DL resume source (archives from an earlier run) + if: github.event_name == 'workflow_dispatch' && inputs.dl_source_run_id != '' + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + version: ${{ steps.src.outputs.version }} + archive_url: ${{ steps.src.outputs.archive_url }} + archive_sha256: ${{ steps.src.outputs.archive_sha256 }} + runtime_url: ${{ steps.src.outputs.runtime_url }} + runtime_sha256: ${{ steps.src.outputs.runtime_sha256 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download the source run's manifest artifact + env: + GH_TOKEN: ${{ github.token }} + run: gh run download "${{ inputs.dl_source_run_id }}" -R "${{ github.repository }}" -n dl-manifest -D dl-manifest + + - name: Resolve the archives the manifest names; refuse a version mismatch or a published manifest + id: src + run: | + set -o pipefail + VERSION=$(node -p "require('./package.json').version") + M=dl-manifest/manifest.json + MV=$(jq -r .version "$M") + test "$MV" = "$VERSION" || { echo "::error::manifest is for $MV but this checkout is $VERSION"; exit 1; } + BASE="https://apps.lloyal.ai/v1/binaries/lloyal.node/$VERSION" + A=$(jq -r .archive.file "$M"); R=$(jq -r .runtimeArchive.file "$M") + for f in "$A" "$R"; do + curl -sfI "$BASE/$f" > /dev/null || { echo "::error::$BASE/$f is not on the channel — nothing to resume"; exit 1; } + done + if curl -sfI "$BASE/linux-x64-dl.manifest.json" > /dev/null; then + echo "::error::a signed manifest is already published for $VERSION"; exit 1 + fi + { + echo "version=$VERSION" + echo "archive_url=$BASE/$A" + echo "archive_sha256=$(jq -r .archive.sha256 "$M")" + echo "runtime_url=$BASE/$R" + echo "runtime_sha256=$(jq -r .runtimeArchive.sha256 "$M")" + } >> "$GITHUB_OUTPUT" + + # publish-dl downloads `dl-manifest` from THIS run, whichever job made it. + - name: Carry the manifest as this run's artifact + uses: actions/upload-artifact@v4 + with: + name: dl-manifest + path: dl-manifest/manifest.json + retention-days: 1 + gpu-tests-dl: name: GPU Tests DL (L4, path-2 consumer) - needs: build-dl-flavor - if: needs.build-dl-flavor.outputs.uploaded == 'true' + needs: [build-dl-flavor, dl-source] + if: ${{ !cancelled() && (needs.build-dl-flavor.outputs.uploaded == 'true' || needs.dl-source.result == 'success') }} runs-on: ubuntu-latest permissions: contents: read @@ -629,15 +709,15 @@ jobs: GCP_REGION: ${{ secrets.GCP_REGION }} GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }} JOB_NAME: lloyal-gpu-test-dl - EXTRA_ENV_VARS: "LLOYAL_PACK_URL=${{ needs.build-dl-flavor.outputs.archive_url }},LLOYAL_PACK_SHA256=${{ needs.build-dl-flavor.outputs.archive_sha256 }},LLOYAL_RUNTIME_URL=${{ needs.build-dl-flavor.outputs.runtime_url }},LLOYAL_RUNTIME_SHA256=${{ needs.build-dl-flavor.outputs.runtime_sha256 }}" + EXTRA_ENV_VARS: "LLOYAL_PACK_URL=${{ needs.build-dl-flavor.outputs.archive_url || needs.dl-source.outputs.archive_url }},LLOYAL_PACK_SHA256=${{ needs.build-dl-flavor.outputs.archive_sha256 || needs.dl-source.outputs.archive_sha256 }},LLOYAL_RUNTIME_URL=${{ needs.build-dl-flavor.outputs.runtime_url || needs.dl-source.outputs.runtime_url }},LLOYAL_RUNTIME_SHA256=${{ needs.build-dl-flavor.outputs.runtime_sha256 || needs.dl-source.outputs.runtime_sha256 }}" # The publish act: sign + create-once the manifest — until this runs, # the uploaded archives are unverifiable ⇒ effectively unpublished. Any # red job upstream ⇒ no signed manifest for this version; npm unaffected. publish-dl: name: Publish DL manifest (signed) - needs: [build-dl-flavor, gpu-tests-dl] - if: needs.build-dl-flavor.outputs.uploaded == 'true' + needs: [build-dl-flavor, dl-source, gpu-tests-dl] + if: ${{ !cancelled() && needs.gpu-tests-dl.result == 'success' && (needs.build-dl-flavor.outputs.uploaded == 'true' || needs.dl-source.result == 'success') }} runs-on: ubuntu-latest steps: # Explicit path: correctness here otherwise rests on two chained @@ -659,7 +739,7 @@ jobs: # would feed empty input to the trailing jq, which exits 0, and # the SIGNING act would report success without publishing. set -o pipefail - VERSION="${{ needs.build-dl-flavor.outputs.version }}" + VERSION="${{ needs.build-dl-flavor.outputs.version || needs.dl-source.outputs.version }}" jq -n --arg v "$VERSION" --slurpfile m dl-manifest/manifest.json \ '{version: $v, platform: "linux-x64-dl", manifest: $m[0]}' \ | curl -sfS -X POST https://api.lloyal.ai/v1/binaries/publish \ @@ -672,8 +752,14 @@ jobs: name: Publish all packages needs: [build-and-test, gpu-tests] runs-on: ubuntu-latest - # Only run if ALL jobs succeeded AND either it's a tag push or manual run without skip - if: success() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.skip_publish)) + # Publish when the matrix succeeded and GPU tests either succeeded or were + # deliberately skipped (alpha cuts) — `success()` alone would treat the + # skip as a veto. A GPU FAILURE still blocks. + if: >- + !cancelled() && + needs.build-and-test.result == 'success' && + (needs.gpu-tests.result == 'success' || needs.gpu-tests.result == 'skipped') && + (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.skip_publish)) permissions: contents: read id-token: write # Required for npm provenance diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a1881f4..f6b7dbf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,6 +67,14 @@ jobs: # cache resolution, and the pack extractor. Plus the arch-mirror # drift gate: fails when a llama.cpp sync changes ggml's CUDA arch # derivation without scripts/dl-archs.js being reconciled. + # + # The test entries run against dist, the output that ships — + # integration.ts always did, and the unit test does since 6962f5c + # (importing ../src from a test made the compile gate emit .js beside + # the sources). tsc only; no native build is needed for this. + - name: Build TypeScript + run: npm run build:ts + - name: Backend-pack unit tests + arch drift gate + stderr allowlist if: runner.os == 'Linux' run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c785f4..33479fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,6 +130,26 @@ if(MSVC) endif() endif() +# ============================================================================= +# mtmd (multimodal: clip/projector encode + media tokenization) +# ============================================================================= +# tools/mtmd is not built by llama.cpp under add_subdirectory (LLAMA_BUILD_TOOLS +# is OFF for non-standalone builds), so add the library directly. +# - EXCLUDE_FROM_ALL: mtmd's CMakeLists also declares CLI targets +# (llama-mtmd-cli etc.); only the linked `mtmd` library gets built. +# - LLAMA_INSTALL_VERSION is a normal variable scoped inside llama.cpp's +# directory — empty here, and mtmd's set_target_properties(VERSION ...) +# would receive "". Provide it before the add. +# - MTMD_VIDEO is a compile definition only (video decoding shells out to an +# ffmpeg binary at runtime); keep it OFF for a deterministic surface. +set(LLAMA_INSTALL_VERSION 0.0.0) +set(MTMD_VIDEO OFF CACHE BOOL "mtmd video support (runtime ffmpeg)" FORCE) +add_subdirectory(${LLAMA_CPP_DIR}/tools/mtmd mtmd EXCLUDE_FROM_ALL) +if(MSVC AND TARGET mtmd) + # Same u8"" literal workaround as the llama.cpp targets above. + target_compile_options(mtmd PRIVATE /Zc:char8_t-) +endif() + # Build liblloyal (INTERFACE library, links llama transitively) # This also sets up the llama/llama.h include structure automatically add_subdirectory(${LIBLLOYAL_DIR} liblloyal) @@ -202,6 +222,7 @@ endif() target_link_libraries(${PROJECT_NAME} PRIVATE liblloyal::liblloyal ${LLAMA_COMMON_TARGET} + mtmd md4c ${CMAKE_JS_LIB} ) @@ -288,7 +309,7 @@ if(BUILD_SHARED_LIBS) # libggml-cuda alone). Symlinks are intentionally avoided: they require admin / # Developer Mode on Windows (see liblloyal/CMakeLists.txt). This block is # Linux/macOS only (Windows links statically; SONAMEs don't apply to DLLs). - foreach(_lib IN ITEMS llama ${LLAMA_COMMON_TARGET} ggml ggml-base ggml-cpu ggml-metal ggml-cuda ggml-vulkan ggml-blas) + foreach(_lib IN ITEMS llama ${LLAMA_COMMON_TARGET} mtmd ggml ggml-base ggml-cpu ggml-metal ggml-cuda ggml-vulkan ggml-blas) if(TARGET ${_lib}) get_target_property(_lloyal_lib_type ${_lib} TYPE) if(_lloyal_lib_type STREQUAL "SHARED_LIBRARY") diff --git a/README.md b/README.md index 6f7e68e..5da85b8 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,17 @@ [![GPU Tests](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml/badge.svg)](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml) [![npm](https://img.shields.io/npm/v/@lloyal-labs/lloyal.node.svg)](https://www.npmjs.com/package/@lloyal-labs/lloyal.node) [![License](https://img.shields.io/badge/license-FSL--1.1--Apache--2.0-blue.svg)](LICENSE) -[![llama.cpp](https://img.shields.io/badge/llama.cpp-b8795-green.svg)](https://github.com/ggml-org/llama.cpp/releases/tag/b8795) +[![llama.cpp](https://img.shields.io/badge/llama.cpp-pinned-green.svg)](./liblloyal/.llama-cpp-version) -**Native backend for the lloyal inference platform.** +**The Node runtime for the HDK — built on liblloyal and llama.cpp** -Prebuilt llama.cpp binaries for 13 platform/GPU combinations, exposing a `SessionContext` that powers the [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) inference primitives (Branch, BranchStore, Session, Rerank) and [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) multi-agent framework. Built on [liblloyal](https://github.com/lloyal-ai/liblloyal), a header-only C++20 inference kernel for llama.cpp. +The HDK's packages are backend-agnostic TypeScript: `Branch` and `BranchStore` in the SDK, agents and pools above them. They describe what to do with inference state; they cannot decode a token on their own. On Node, this package is what they run on. -All SDK and agent exports are re-exported from this package for convenience — `import { Branch, useAgent, agentPool } from "@lloyal-labs/lloyal.node"` works out of the box. +`createContext()` returns the `SessionContext` every HDK primitive drives — forking a branch, batching N branches into one dispatch, prefilling text or image rows into a branch's KV. Underneath it is [liblloyal](https://github.com/lloyal-ai/liblloyal), the header-only C++20 kernel that enables Git-like tree ops for live inference state, on llama.cpp for the model execution. + +It ships **prebuilt for 13 platform/GPU targets**. Nothing compiles on install, and the variant matching your hardware is chosen when the process starts — so the same artifact ships to a laptop and a CUDA box. + +The rest of the HDK is re-exported, so `import { Branch, useAgent } from "@lloyal-labs/lloyal.node"` needs no second package. ## Install @@ -18,8 +22,6 @@ All SDK and agent exports are re-exported from this package for convenience — npm install @lloyal-labs/lloyal.node ``` -Prebuilt binaries for 13 platform/GPU combinations. GPU selection at runtime, not install time. - | Platform | Arch | Acceleration | | -------- | ----- | ------------------- | | macOS | arm64 | Metal | @@ -29,7 +31,7 @@ Prebuilt binaries for 13 platform/GPU combinations. GPU selection at runtime, no | Windows | x64 | CPU / CUDA / Vulkan | | Windows | arm64 | CPU / Vulkan | -## Quick Start +## Quick start ```javascript import { createContext } from "@lloyal-labs/lloyal.node"; @@ -41,162 +43,230 @@ const store = new BranchStore(ctx); const root = Branch.create(ctx, 0, { temperature: 0.8 }); await root.prefill(await ctx.tokenize("Explain quantum entanglement")); -// Fork and generate — all branches in lockstep, 1 GPU call per step +// Fork three ways; every live branch advances in one GPU call per step const branches = await Promise.all([root.fork(), root.fork(), root.fork()]); for (;;) { const live = branches.filter((b) => !b.disposed); if (!live.length) break; - const produced = live.map((b) => ({ b, ...b.produce() })); + + const produced = live.map((b) => ({ b, ...b.produceSync() })); for (const p of produced.filter((p) => p.isStop)) await p.b.prune(); + const items = produced .filter((p) => !p.isStop) - .map((p) => { - p.b.accept(p.token); - return [p.b, p.token]; - }); - await store.commit(items); + .map((p) => [p.b, p.token]); + if (items.length) await store.commit(items); // accept + decode: N branches, 1 llama_decode() } ``` -Or for single-branch generation, Branch is an async iterable: +`produceSync()` samples without awaiting so the whole cohort can be collected and committed together — that batching is the point. `await branch.produce()` is the single-branch form. + +For one branch, `Branch` is an async iterable: ```javascript -for await (const { token, text } of branch) { - process.stdout.write(text); -} +for await (const { token, text } of branch) process.stdout.write(text); ``` -See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) for the full Branch API, continuous tree batching, KV tenancy, and topology documentation. +See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) for the Branch API, continuous tree batching, KV tenancy and topology. -### Without the SDK +## What this package is -`createContext` returns a `SessionContext` — the native interface to llama.cpp. You can use it directly without the SDK's Branch/BranchStore layer: +lloyal.node binds [liblloyal](https://github.com/lloyal-ai/liblloyal) — the C++20 kernel — to Node, and ships it prebuilt. It is the seam: everything above it is backend-agnostic TypeScript, everything below is native. That is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. -```javascript -import { createContext } from "@lloyal-labs/lloyal.node"; +**What it owns:** + +- `createContext(options)` — load a GGUF, get a `SessionContext`. `mmprojPath` loads a multimodal projector beside it. +- `_storePrefillMultimodal(...)` — image + text into a branch's KV, plus `supportsVision()` / `supportsAudio()` +- `loadBinary(variant?)` and the [binary resolution order](#which-binary-loads) +- The prebuilt binaries and the [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) + +**What it re-exports**, so one install is enough — these are [HDK](https://github.com/lloyal-ai/hdk) packages, documented there: +- from `@lloyal-labs/sdk`: `Branch`, `BranchStore`, `Session`, `Rerank`, `buildUserDelta`, `buildToolResultDelta`, and the sampling, chat and rerank types +- from `@lloyal-labs/lloyal-agents`: `Tool`, `Agent`, `agent`, `agentPool`, `useAgent`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, `initAgents`, `DefaultAgentPolicy`, `renderTemplate` + +Not re-exported — import from the package itself: the Ability protocol (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`, `GrantStoreCtx`) from `@lloyal-labs/lloyal-agents`, and `defineAbility` / `createAbilityRegistry` / `createGrantStore` from `@lloyal-labs/rig`. + +## The native surface + +`createContext` returns a `SessionContext` — llama.cpp as this package exposes it. The SDK's `Branch`/`BranchStore` wrap these; you can use them directly. + +```javascript const ctx = await createContext({ modelPath: "./model.gguf", nSeqMax: 4 }); -// Chat templates — model-agnostic formatting + tool calling -const { prompt, grammar, format } = await ctx.formatChat(messages, { - addGenerationPrompt: true, - tools: [{ type: "function", function: { name: "search", parameters: schema } }], -}); +// Chat templates — model-agnostic formatting and tool calling. +// NOTE: messages go in as a JSON STRING, not an object. +const { prompt, grammar, format } = await ctx.formatChat( + JSON.stringify([{ role: "user", content: "hello" }]), + { addGenerationPrompt: true, + tools: [{ type: "function", function: { name: "search", parameters: schema } }] }, +); const { content, toolCalls } = await ctx.parseChatOutput(output, format); -// Branch primitives — what the SDK's Branch class wraps +// Branch primitives — what Branch wraps const handle = ctx._branchCreate(0, samplerParams); await ctx._branchPrefill(handle, tokens); const token = ctx._branchSample(handle); -const text = ctx.tokenToText(token); -const isStop = ctx.isStopToken(token); ctx._branchAccept(handle, token); -const logits = ctx._branchGetLogits(handle); // Float32Array(vocabSize) -const entropy = ctx._branchModelEntropy(handle); +const logits = ctx._branchGetLogits(handle); // Float32Array(vocabSize) const child = ctx._branchFork(handle); -// Store primitives — what the SDK's BranchStore wraps -await ctx._storeCommit([handle1, handle2], [tok1, tok2]); // N branches, 1 GPU call +// Store primitives — what BranchStore wraps +await ctx._storeCommit([handle1, handle2], [tok1, tok2]); // N branches, 1 GPU call await ctx._storePrefill([handle], [tokens]); await ctx._storeRetainOnly(winner); -const available = ctx._storeAvailable(); - -// KV cache — snapshot, copy, persist -await ctx.kvSeqCopy(0, 1); // share prefix across sequences -await ctx.kvCacheSave(); // snapshot for rollback -await ctx.kvCacheLoad(); // restore checkpoint -await ctx.kvCacheWriteFile("cache.bin"); // persist to disk -// Embeddings +// KV, embeddings, grammar +await ctx.kvSeqCopy(0, 1); const embeddings = await ctx.encode("query text"); -const dim = ctx.getEmbeddingDimension(); - -// Grammar + tokenizer const grammar = await ctx.jsonSchemaToGrammar(schema); -const tokens = await ctx.tokenize("Hello world"); -const sep = await ctx.getTurnSeparator(); ``` -## What This Package Provides +## Multimodal -**Native-only** (not in SDK): +An image is decoded, projected into the model's native input embeddings, and admitted through `llama_batch.embd` beside the token stream. After that it is ordinary KV: fork the branch and every child attends the image with no re-encode. -- `createContext(options)` — load a GGUF model, return a `SessionContext` -- `loadBinary(options?)` — explicit GPU variant selection with automatic fallback -- Prebuilt binaries for 13 platform/GPU combinations +```javascript +const ctx = await createContext({ + modelPath: "./Qwen3.5-4B-Q4_K_M.gguf", + mmprojPath: "./mmproj-F16.gguf", + nSeqMax: 8, +}); +ctx.supportsVision(); // true + +// One <__media__> marker per image, as a media_marker content part +const { prompt } = await ctx.formatChat(JSON.stringify([ + { role: "user", content: [ + { type: "text", text: "What is in this image?" }, + { type: "media_marker", text: "<__media__>" }, + ]}, +])); + +const handle = ctx._branchCreate(0, { temperature: 0 }); +const bytes = fs.readFileSync("./photo.jpg"); // jpg/png/bmp/gif +const [{ tokensDecoded, positionAdvance }] = + await ctx._storePrefillMultimodal([handle], [[]], [prompt], [[bytes]]); +``` + +`positionAdvance < tokensDecoded` under M-RoPE — an image costs more KV cells than it advances position, and the kernel tracks the gap so pressure accounting stays exact. Several markers with several images in one prefill also works; video frames with timestamps are exactly that. + +> **Types.** `createContext` is typed as `ContextOptions` from `@lloyal-labs/sdk`, so `mmprojPath` / `imageMinTokens` / `imageMaxTokens` and the `supportsVision()` / `_storePrefillMultimodal()` members only typecheck once an SDK carrying multimodal `ContextOptions` is installed. The runtime accepts them regardless. -**Re-exported from [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk):** +A configured `mmprojPath` that fails to load throws at `createContext` — never a silent fall back to text-only. Audio is rejected explicitly. -- `Branch`, `BranchStore`, `Session`, `Rerank` -- Per-token metrics: `modelEntropy()`, `modelSurprisal()`, `samplingPerplexity` -- Chat formatting: `formatChat()`, `parseChatOutput()` -- Grammar: `jsonSchemaToGrammar()`, `setGrammar()` +## Which binary loads -**Re-exported from [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents):** +Resolution is ordered and mostly invisible — but when the wrong binary loads, this is the order that decided it. -- `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit` -- Structured concurrency DAG via Effection generators -- In-loop orchestration: agents as branches of a single running process -- App protocol surfaces (`AppRegistryCtx`, `AppConfigStoreCtx`, `App`, `AppManifest`) when paired with [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig)'s `defineApp` / `createAppRegistry` +| # | Source | If it fails | +| --- | --- | --- | +| 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | +| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws; asserts the devices you asked for | +| 3 | a cached [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws if present-but-invalid — **no** fallthrough to npm | +| 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | +| 5 | local `build/Release` | continues — fresher than an installed package during development | +| 6 | default CPU package for the platform | throws, naming everything it tried | -## GPU Variant Selection + +## GPU variant selection ```javascript import { loadBinary, createContext } from "@lloyal-labs/lloyal.node"; -// Automatic — uses Metal on macOS, CPU elsewhere +// Automatic — Metal on macOS arm64, CPU elsewhere const ctx = await createContext({ modelPath: "./model.gguf" }); -// Explicit CUDA -const binding = loadBinary({ gpuVariant: "cuda" }); -const ctx = await binding.createContext({ modelPath: "./model.gguf" }); -// Falls back to CPU with a warning if CUDA runtime not available +// Explicit: loadBinary takes the variant directly +const binding = loadBinary("cuda"); // "default" | "cuda" | "vulkan" +const ctx2 = await binding.createContext({ modelPath: "./model.gguf" }); +// Falls back to CPU with a warning unless LLOYAL_NO_FALLBACK=1 +``` + +## The backend pack — frontier GPUs, and every CPU + +The npm packages are one build per platform/GPU pair. The backend pack is the other shape: **one artifact carrying many backends as separately loadable modules** (`GGML_BACKEND_DL`), chosen at load time. It is a full `lloyal.node` addon plus its backends, not a set of loose libraries. + +That buys two things. + +**Frontier GPUs.** Blackwell is newer than most published builds, so an **sm_100** device would otherwise fall back to JIT or to CPU. The pack ships real SASS for it, and a per-arch `cuobjdump` gate at publish time refuses to build one where any declared arch is missing its SASS or PTX — the claim cannot silently rot. + +**Every CPU microarchitecture.** Built with `GGML_CPU_ALL_VARIANTS`, gated at **≥8** `libggml-cpu-.so` modules, so the best instruction set for the host is picked at load rather than baked in. This is not only a GPU feature. + +```javascript +import { probeBackendPack, ensureBackendPack } from "@lloyal-labs/lloyal.node"; + +const offer = await probeBackendPack(); // inspects only — never downloads +if (offer.recommended) await ensureBackendPack(); ``` +**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — a pack arrives only through an explicit `ensureBackendPack()` or a provisioner. On load it also calls `listDevices()` and **throws if a GPU was requested and none registered**, so a pack that quietly came up CPU-only fails loudly instead of being slow. + +Three gates run before a pack is offered at all: + +| Gate | Question | +| --- | --- | +| device | is the GPU covered by real SASS, or by a JIT-able PTX floor? | +| driver | native SASS needs no JIT; otherwise, can the driver JIT the pack's toolkit PTX? | +| runtime | does the installed CUDA runtime meet the manifest's minimum, or is the companion runtime needed too? | + +| GPU | Outcome | +| --- | --- | +| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion archive with it | +| H100 (sm_90) | PTX only — offered where the driver can JIT | +| L4 (sm_89) | never offered; the npm package already ships native for it | +| no NVIDIA GPU | never offered | + +The companion runtime is its own archive — `cudart`, `cublas`, `cublasLt`, `nvJitLink` — so a host with an older CUDA can still run the pack without touching its system install. + +Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than falling through to npm, which is why it sits above the variant lookup in the table above. + +> **Two channels, not one.** The 13 prebuilt npm packages cover macOS, Linux **and Windows** — `win32-x64-cuda` is one of them, so Windows CUDA ships that way and needs nothing from this section. +> +> The backend pack is a separate, opt-in channel published only for **linux-x64**. `platformTag()` returns `null` anywhere else, so a pack is never even looked for, and `LLOYAL_BACKEND_DL=1` refuses to build one. linux-arm64 is the named follow-on. + ## Examples | Example | Pattern | | --------------------------------- | ------------------------------------------------- | -| [`entropy/`](./examples/entropy/) | `modelEntropy()` mid-generation as control signal | | [`chat/`](./examples/chat/) | Interactive streaming chat | -| [`embed/`](./examples/embed/) | Text embeddings extraction | +| [`embed/`](./examples/embed/) | Text embedding extraction | +| [`entropy/`](./examples/entropy/) | `modelEntropy()` mid-generation as a control signal | ```bash -npx tsx examples/best-of-n/best-of-n.ts npx tsx examples/chat/chat.ts ./model.gguf ``` -## CI Testing +## CI -Integration tests run real inference across architectures: +Integration tests run real inference across architectures, so a template regression surfaces as a wrong answer rather than a clean pass: -| Architecture | Test Model | Template | -| ------------ | ------------ | -------- | -| Llama | Llama 3.2 1B | llama3 | -| Phi | Phi 3.5 Mini | phi3 | -| Qwen | Qwen 3 1.7B | chatml | -| Gemma | Gemma 3 1B | gemma | -| SmolLM | SmolLM2 1.7B | chatml | -| Ministral | Ministral 3B | mistral | +| Model | Template | +| ------------ | ---------- | +| SmolLM2 1.7B | chatml *(default)* | +| Llama 3.2 | llama3 | +| Phi 3.5 | phi3 | +| Qwen3 | chatml | +| Gemma 3 | gemma | +| GLM-Edge | glm-edge | -See [distribution.md](docs/distribution.md) for details. +Multimodal runs two tiers: SmolVLM-256M for plain positions in CI, Qwen3.5-4B + mmproj for M-RoPE locally and on the GPU rig. See [distribution.md](docs/distribution.md). ## Ecosystem -| Package | Description | -| ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | -| [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives (Branch, BranchStore, Session, Rerank) | -| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + App protocol primitives | -| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | App protocol helpers, retrieval providers, framework tools (Plan/Delegate/Report) | -| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses + Apps; publish/install signed Apps via the channel | -| [liblloyal](https://github.com/lloyal-ai/liblloyal) | Header-only C++20 inference kernel for llama.cpp | -| **lloyal.node** | This package — native backend + prebuilt binaries | -| [nitro-llama](https://github.com/lloyal-ai/nitro-llama) | React Native backend via Nitro Modules | -| [tsampler](https://github.com/lloyal-ai/tsampler) | Reference sampler implementation | +| Package | Description | +| --- | --- | +| [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives | +| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime; owns the Ability protocol contracts and the `GrantStore` / `authGuard` surface that gates `protected` tools | +| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | Builds Abilities on those contracts — `defineAbility`, `createAbilityRegistry`, retrieval and framework tools, and `createGrantStore`, the reference in-memory grant store | +| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses and Abilities, publish/install signed Abilities | +| [liblloyal](https://github.com/lloyal-ai/liblloyal) | The C++20 kernel | +| **lloyal.node** | This package — native backend + prebuilt binaries | +| [nitro-llama](https://github.com/lloyal-ai/nitro-llama) | React Native backend via Nitro Modules | +| [tsampler](https://github.com/lloyal-ai/tsampler) | Reference sampler implementation | ## Contributing -See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup and release process. +See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup and the release process. ## License diff --git a/liblloyal b/liblloyal index a3558a0..d525b22 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit a3558a0619309ca41b7e044bef7329f653273527 +Subproject commit d525b224741fcaf438a7db43198ec8be33ff2612 diff --git a/package-lock.json b/package-lock.json index 71ea16a..9947a46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", + "version": "3.2.0-alpha.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", + "version": "3.2.0-alpha.3", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@lloyal-labs/tsampler": "^0.2.0", @@ -16,8 +16,10 @@ "@lloyal-labs/lloyal-agents": "^3.0.0", "@lloyal-labs/sdk": "^3.0.0", "@types/node": "^25.3.0", + "@types/semver": "^7.8.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", + "semver": "^7.8.5", "tsx": "^4.21.0", "typedoc": "^0.28.16", "typedoc-rhineai-theme": "^1.2.0", @@ -27,23 +29,23 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.1.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.3" }, "peerDependencies": { - "@lloyal-labs/lloyal-agents": ">=3.0.0", - "@lloyal-labs/sdk": ">=3.0.0" + "@lloyal-labs/lloyal-agents": ">=3.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/sdk": ">=3.0.0 || >=4.0.0-0 <5.0.0" } }, "node_modules/@esbuild/aix-ppc64": { @@ -658,6 +660,13 @@ "undici-types": "~7.19.0" } }, + "node_modules/@types/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -1314,9 +1323,9 @@ } }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { diff --git a/package.json b/package.json index f79da1e..561dbbd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", - "description": "Node.js client for liblloyal+llama.cpp", + "version": "3.2.0-alpha.3", + "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", "gypfile": false, @@ -20,7 +20,7 @@ "version": "node scripts/sync-versions.js && git add -A", "docs": "npx typedoc", "test": "npm run test:integration", - "test:unit": "npx tsx test/backend-pack-unit.ts", + "test:unit": "npx tsx test/backend-pack-unit.ts && npx tsx test/peers-unit.ts", "test:canary": "npx tsx test/stderr-canary.ts", "test:integration": "npx tsx test/integration.ts", "test:examples": "npx tsx test/examples.ts", @@ -52,34 +52,36 @@ "node-addon-api": "^8.5.0" }, "peerDependencies": { - "@lloyal-labs/lloyal-agents": ">=3.0.0", - "@lloyal-labs/sdk": ">=3.0.0" + "@lloyal-labs/lloyal-agents": ">=3.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/sdk": ">=3.0.0 || >=4.0.0-0 <5.0.0" }, "devDependencies": { "@lloyal-labs/lloyal-agents": "^3.0.0", "@lloyal-labs/sdk": "^3.0.0", "@types/node": "^25.3.0", + "@types/semver": "^7.8.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", + "semver": "^7.8.5", "tsx": "^4.21.0", "typedoc": "^0.28.16", "typedoc-rhineai-theme": "^1.2.0", "typescript": "^5.9.3" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.1.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.3" }, "engines": { "node": ">=24.0.0" diff --git a/scripts/check-stderr-allowlist.js b/scripts/check-stderr-allowlist.js index e243391..1bd6972 100644 --- a/scripts/check-stderr-allowlist.js +++ b/scripts/check-stderr-allowlist.js @@ -25,7 +25,7 @@ const SRC = path.join(__dirname, '..', 'src'); /** file (relative to src/) → exact number of writer call sites. */ const ALLOWLIST = { 'BackendManager.cpp': 2, // once-guarded backend-init provenance + fatal dladdr failure - 'SessionContext.cpp': 13, // initializeContext (4) + CreateContext (9), all boot-scoped + 'SessionContext.cpp': 15, // initializeContext (4) + initializeMultimodal (1) + CreateContext (10), all boot-scoped }; // std::cerr / std::cout streams, fprintf(stderr|stdout, and bare printf — diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index cdc4f22..5e823be 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -624,16 +626,30 @@ class StoreCommitWorker : public Napi::AsyncWorker { } // ~Snapshot frees the swapped-out (post-accept) state + // One arm, dynamic_cast for the rc — no catch-order hazard. The typed + // exception never crosses N-API; the rc travels as data (see OnError). + if (const auto* de = dynamic_cast(&e)) { _rc = de->rc; _partial = de->partial; } SetError(e.what()); } } void OnOK() override { _deferred.Resolve(Env().Undefined()); } - void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + void OnError(const Napi::Error& err) override { + // Attach the rc to the JS error we construct — a property on an object we + // own, set on the JS thread. This is the whole boundary crossing. + if (_rc != 0) { + auto o = err.Value().As(); + o.Set("rc", Napi::Number::New(Env(), _rc)); + o.Set("partial", Napi::Boolean::New(Env(), _partial)); + } + _deferred.Reject(err.Value()); + } Napi::Promise GetPromise() { return _deferred.Promise(); } private: Napi::Promise::Deferred _deferred; + int32_t _rc = 0; + bool _partial = false; lloyal::branch::BranchStore& _store; std::vector _items; }; @@ -660,20 +676,139 @@ class StorePrefillWorker : public Napi::AsyncWorker { items[i].tokens = _tokenStorage[i]; } _store.decode_scatter(items); - } catch (const std::exception& e) { SetError(e.what()); } + } catch (const std::exception& e) { + // One arm, dynamic_cast for the rc — no catch-order hazard. + if (const auto* de = dynamic_cast(&e)) { _rc = de->rc; _partial = de->partial; } + SetError(e.what()); + } } void OnOK() override { _deferred.Resolve(Env().Undefined()); } - void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + void OnError(const Napi::Error& err) override { + if (_rc != 0) { + auto o = err.Value().As(); + o.Set("rc", Napi::Number::New(Env(), _rc)); + o.Set("partial", Napi::Boolean::New(Env(), _partial)); + } + _deferred.Reject(err.Value()); + } Napi::Promise GetPromise() { return _deferred.Promise(); } private: Napi::Promise::Deferred _deferred; + int32_t _rc = 0; + bool _partial = false; lloyal::branch::BranchStore& _store; std::vector _handles; std::vector> _tokenStorage; }; +/** + * AsyncWorker for multimodal prefill — a marshaller, like its siblings. + * + * Owns everything it touches off-thread (prompt strings, copied image bytes, + * sep tokens) and hands each branch to the kernel as one call. The pipeline + * itself lives where it belongs: `lloyal::MtmdSource` (mtmd.hpp) produces the + * segment stream, `BranchStore::decode_segments` places it — rails, positions, + * logits and cell/slack accounting are all the store's, so no KV state + * crosses into this layer. + * + * Returns per-branch {tokensDecoded, positionAdvance}: JS cannot know + * multimodal token counts (mtmd owns tokenization) and the trace + pressure + * layers need them. + */ +class StorePrefillMultimodalWorker : public Napi::AsyncWorker { +public: + struct BranchResult { + int64_t tokensDecoded = 0; + int64_t positionAdvance = 0; + /** Empty when this entry landed. Non-empty ⇒ the entry FAILED — either + * before its decode ran (bitmap decode, marker/bitmap mismatch: the + * branch is untouched) or inside decode_segments. `rc` and `partial` + * say which case a decode failure is (see DecodeError in liblloyal): + * intact iff the failing call restored state (rc 1 or -1) and nothing + * before it landed (!partial); anything else ⇒ prune and replay from + * content — pruning an untouched branch is safe. */ + std::string error; + /** llama_decode's raw return code when the failure carried one (0 + * otherwise — validation throws never reached llama_decode). */ + int32_t rc = 0; + /** True when an earlier chunk of this entry landed before the failing + * call — the branch is then not intact even when `rc` restored state. */ + bool partial = false; + }; + + StorePrefillMultimodalWorker(Napi::Env env, + lloyal::branch::BranchStore& store, + mtmd_context* mtmd, + int32_t nEmbdInp, + std::vector handles, + std::vector> sepStorage, + std::vector prompts, + std::vector>> bitmapBytes) + : AsyncWorker(env), _deferred(env), _store(store), _mtmd(mtmd), + _nEmbdInp(nEmbdInp), _handles(std::move(handles)), + _sepStorage(std::move(sepStorage)), _prompts(std::move(prompts)), + _bitmapBytes(std::move(bitmapBytes)) {} + + // Per-entry isolation, deliberately: a corrupt image in one agent's tool + // result must not cost its siblings their prefills. Failing the whole call + // would also lose WHICH entries landed, and the caller needs that to prune + // exactly the poisoned branches. Only a failure outside the loop (an + // allocation) can still reject the promise, because then there are no + // per-entry results to report at all. + void Execute() override { + try { + _results.resize(_handles.size()); + } catch (const std::exception& e) { SetError(e.what()); return; } + + for (size_t i = 0; i < _handles.size(); ++i) { + try { + lloyal::MtmdSource source( + _mtmd, _prompts[i], _bitmapBytes[i], + std::span(_sepStorage[i]), _nEmbdInp); + const auto r = _store.decode_segments(_handles[i], source); + _results[i] = { r.cells, static_cast(r.advance), "" }; + } catch (const std::exception& e) { + const auto* de = dynamic_cast(&e); + _results[i] = { 0, 0, e.what(), de ? de->rc : 0, de ? de->partial : false }; + } + } + } + + void OnOK() override { + Napi::Env env = Env(); + Napi::Array out = Napi::Array::New(env, _results.size()); + for (size_t i = 0; i < _results.size(); ++i) { + Napi::Object r = Napi::Object::New(env); + r.Set("tokensDecoded", Napi::Number::New(env, static_cast(_results[i].tokensDecoded))); + r.Set("positionAdvance", Napi::Number::New(env, static_cast(_results[i].positionAdvance))); + if (!_results[i].error.empty()) { + r.Set("error", Napi::String::New(env, _results[i].error)); + if (_results[i].rc != 0) { + r.Set("rc", Napi::Number::New(env, _results[i].rc)); + r.Set("partial", Napi::Boolean::New(env, _results[i].partial)); + } + } + out.Set(static_cast(i), r); + } + _deferred.Resolve(out); + } + void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + Napi::Promise GetPromise() { return _deferred.Promise(); } + +private: + Napi::Promise::Deferred _deferred; + lloyal::branch::BranchStore& _store; + mtmd_context* _mtmd; + int32_t _nEmbdInp; + std::vector _handles; + std::vector> _sepStorage; + std::vector _prompts; + std::vector>> _bitmapBytes; + std::vector _results; +}; + /** * AsyncWorker for batch logit scoring (process_chunks) * Owns token storage and logit output buffers @@ -767,6 +902,7 @@ Napi::Object SessionContext::Init(Napi::Env env, Napi::Object exports) { Napi::Function func = DefineClass(env, "SessionContext", { // ===== CORE ===== InstanceMethod("tokenToText", &SessionContext::tokenToText), + InstanceMethod("tokenToBytes", &SessionContext::tokenToBytes), InstanceMethod("isStopToken", &SessionContext::isStopToken), InstanceMethod("getEogToken", &SessionContext::getEogToken), InstanceMethod("getTurnSeparator", &SessionContext::getTurnSeparator), @@ -841,6 +977,10 @@ Napi::Object SessionContext::Init(Napi::Env env, Napi::Object exports) { // ===== STORE API (internal, wrapped by lib/BranchStore.js) ===== InstanceMethod("_storeCommit", &SessionContext::_storeCommit), InstanceMethod("_storePrefill", &SessionContext::_storePrefill), + InstanceMethod("_storePrefillMultimodal", &SessionContext::_storePrefillMultimodal), + InstanceMethod("_cellsMultimodal", &SessionContext::_cellsMultimodal), + InstanceMethod("supportsVision", &SessionContext::supportsVision), + InstanceMethod("supportsAudio", &SessionContext::supportsAudio), InstanceMethod("_storeMergeLogits", &SessionContext::_storeMergeLogits), InstanceMethod("_storeRetainOnly", &SessionContext::_storeRetainOnly), InstanceMethod("_storeAvailable", &SessionContext::_storeAvailable), @@ -873,6 +1013,11 @@ SessionContext::SessionContext(const Napi::CallbackInfo& info) SessionContext::~SessionContext() { if (!_disposed) { + // Free mtmd first — it holds a reference to the model + if (_mtmdContext) { + mtmd_free(_mtmdContext); + _mtmdContext = nullptr; + } // Free context (depends on model) if (_context) { llama_free(_context); @@ -898,6 +1043,14 @@ void SessionContext::initializeContext( std::cerr << " Shared refcount: " << _model.use_count() << std::endl; } +void SessionContext::initializeMultimodal(struct mtmd_context* mtmd) { + _mtmdContext = mtmd; + std::cerr << "[SessionContext::initializeMultimodal] mmproj attached" + << " (vision=" << (mtmd_support_vision(mtmd) ? "yes" : "no") + << ", audio=" << (mtmd_support_audio(mtmd) ? "yes" : "no") << ")" + << std::endl; +} + Napi::Value SessionContext::tokenize(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); @@ -995,6 +1148,26 @@ Napi::Value SessionContext::tokenToText(const Napi::CallbackInfo& info) { return Napi::String::New(env, text); } +Napi::Value SessionContext::tokenToBytes(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (info.Length() < 1 || !info[0].IsNumber()) { + throw Napi::TypeError::New(env, "Expected token ID (number)"); + } + + llama_token token = static_cast(info[0].As().Int32Value()); + + // Byte-level twin of tokenToText. A BPE piece is a byte sequence that can + // end mid-character, so converting one piece to a JS string can tear + // multi-byte UTF-8 into U+FFFD. The SDK assembles streamed text from bytes + // at character boundaries; the binding stays a passthrough. + std::string text = lloyal::tokenizer::detokenize(_model.get(), token, true); + + return Napi::Buffer::Copy( + env, reinterpret_cast(text.data()), text.size()); +} + // ===== EMBEDDING EXTRACTION ===== Napi::Value SessionContext::encode(const Napi::CallbackInfo& info) { @@ -1263,6 +1436,12 @@ Napi::Value SessionContext::dispose(const Napi::CallbackInfo& info) { // Drain branch store while context is still alive _branchStore.drain(); + // Free mtmd first — it holds a reference to the model + if (_mtmdContext) { + mtmd_free(_mtmdContext); + _mtmdContext = nullptr; + } + // Free context if (_context) { llama_free(_context); @@ -1736,6 +1915,23 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { typeV = t; } + // Extract mmprojPath (optional — enables multimodal input via mtmd) + std::string mmprojPath; + if (options.Has("mmprojPath") && options.Get("mmprojPath").IsString()) { + mmprojPath = options.Get("mmprojPath").As().Utf8Value(); + } + + // Extract image token budget overrides (optional; -1 = model metadata). + // Per-image/per-frame token budgeting — pass-through to mtmd_context_params. + int32_t imageMinTokens = -1; + if (options.Has("imageMinTokens") && options.Get("imageMinTokens").IsNumber()) { + imageMinTokens = options.Get("imageMinTokens").As().Int32Value(); + } + int32_t imageMaxTokens = -1; + if (options.Has("imageMaxTokens") && options.Get("imageMaxTokens").IsNumber()) { + imageMaxTokens = options.Get("imageMaxTokens").As().Int32Value(); + } + // Ensure llama backend is initialized on main thread (thread-safe, once) BackendManager::ensureInitialized(); @@ -1794,6 +1990,45 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { std::cerr << "[CreateContext] Context created successfully" << std::endl; + // ctx and mtmdCtx stay RAW until initializeContext/initializeMultimodal take + // ownership. Everything between here and there can throw — a missing mmproj, + // a failed projector load, ctor.New({}), Unwrap — and each would leak. The + // guard makes cleanup unconditional; release by nulling once ownership moves. + struct NativeHandles { + llama_context* ctx = nullptr; + mtmd_context* mtmd = nullptr; + ~NativeHandles() { + if (mtmd) mtmd_free(mtmd); + if (ctx) llama_free(ctx); + } + } owned; + owned.ctx = ctx; + + // Load the mmproj (multimodal projector) BEFORE the model shared_ptr is + // moved into the instance — mtmd_init_from_file validates the projector's + // output dim against the text model and holds a model reference. Fail + // loud: a configured mmproj that cannot load must never silently degrade + // to text-only. + mtmd_context* mtmdCtx = nullptr; + if (!mmprojPath.empty()) { + std::string fsMmprojPath = liblloyal_node::FileSystem::normalizePath(mmprojPath); + if (!liblloyal_node::FileSystem::exists(fsMmprojPath)) { + throw Napi::Error::New(env, "mmproj file not found: " + fsMmprojPath); + } + mtmd_context_params mparams = mtmd_context_params_default(); + mparams.print_timings = false; + if (nThreads > 0) mparams.n_threads = nThreads; + if (imageMinTokens > 0) mparams.image_min_tokens = imageMinTokens; + if (imageMaxTokens > 0) mparams.image_max_tokens = imageMaxTokens; + std::cerr << "[CreateContext] Loading mmproj: " << fsMmprojPath << std::endl; + mtmdCtx = mtmd_init_from_file(fsMmprojPath.c_str(), sharedModel.get(), mparams); + if (!mtmdCtx) { + throw Napi::Error::New(env, + "Failed to load mmproj (unsupported projector or corrupt file): " + fsMmprojPath); + } + owned.mtmd = mtmdCtx; + } + // Create SessionContext instance Napi::Function ctor = env.GetInstanceData()->Value(); Napi::Object instance = ctor.New({}); @@ -1801,6 +2036,11 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { // Initialize obj->initializeContext(std::move(sharedModel), ctx, nBatch); + owned.ctx = nullptr; // ownership transferred + if (mtmdCtx) { + obj->initializeMultimodal(mtmdCtx); + owned.mtmd = nullptr; + } std::cerr << "[CreateContext] SessionContext initialized" << std::endl; return instance; @@ -2452,6 +2692,219 @@ Napi::Value SessionContext::_storePrefill(const Napi::CallbackInfo& info) { return worker->GetPromise(); } +/** + * Reports what a multimodal prefill WOULD cost, without decoding it. + * + * Admission needs a number before the branch is touched: `decode_segments` is + * not atomic, so a caller that discovers the overflow midway has poisoned the + * branch, while one that refuses up front has spent nothing. Text can be + * measured by tokenizing it; an image cannot, because the caller holds bytes + * and the row count depends on the projector's geometry. + * + * Constructing the source is the measurement: `MtmdSource` counts cells after + * `mtmd_tokenize` and BEFORE any clip encode, so this pays bitmap decode plus + * tokenization — not the vision-tower pass, which stays in the prefill. + */ +class CellsMultimodalWorker : public Napi::AsyncWorker { +public: + CellsMultimodalWorker(Napi::Env env, + mtmd_context* mtmd, + int32_t nEmbdInp, + std::vector sep, + std::string prompt, + std::vector> bitmapBytes) + : AsyncWorker(env), _deferred(env), _mtmd(mtmd), _nEmbdInp(nEmbdInp), + _sep(std::move(sep)), _prompt(std::move(prompt)), + _bitmapBytes(std::move(bitmapBytes)) {} + + void Execute() override { + try { + lloyal::MtmdSource source( + _mtmd, _prompt, _bitmapBytes, + std::span(_sep), _nEmbdInp); + _cells = static_cast(source.cells()); + } catch (const std::exception& e) { SetError(e.what()); } + } + + void OnOK() override { + _deferred.Resolve(Napi::Number::New(Env(), static_cast(_cells))); + } + void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + Napi::Promise GetPromise() { return _deferred.Promise(); } + +private: + Napi::Promise::Deferred _deferred; + mtmd_context* _mtmd; + int32_t _nEmbdInp; + std::vector _sep; + std::string _prompt; + std::vector> _bitmapBytes; + int64_t _cells = 0; +}; + +Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (!_mtmdContext) { + throw Napi::Error::New(env, + "_storePrefillMultimodal requires a multimodal context — pass mmprojPath to createContext"); + } + if (info.Length() < 4 || !info[0].IsArray() || !info[1].IsArray() || + !info[2].IsArray() || !info[3].IsArray()) { + throw Napi::Error::New(env, + "_storePrefillMultimodal requires (handles[], sepTokens[][], prompts[], bitmaps[][])"); + } + + Napi::Array jsHandles = info[0].As(); + Napi::Array jsSeps = info[1].As(); + Napi::Array jsPrompts = info[2].As(); + Napi::Array jsBitmaps = info[3].As(); + const uint32_t n = jsHandles.Length(); + + if (jsSeps.Length() != n || jsPrompts.Length() != n || jsBitmaps.Length() != n) { + throw Napi::Error::New(env, + "_storePrefillMultimodal: all argument arrays must have the same length"); + } + if (n == 0) { + auto deferred = Napi::Promise::Deferred::New(env); + deferred.Resolve(Napi::Array::New(env, 0)); + return deferred.Promise(); + } + + // Marshal everything on the JS thread — the worker owns copies (Buffers + // must never be touched off-thread). + std::vector handles(n); + std::vector> sepStorage(n); + std::vector prompts(n); + std::vector>> bitmapBytes(n); + + for (uint32_t i = 0; i < n; i++) { + handles[i] = static_cast( + jsHandles.Get(i).As().Uint32Value()); + + Napi::Array jsSep = jsSeps.Get(i).As(); + sepStorage[i].resize(jsSep.Length()); + for (uint32_t j = 0; j < jsSep.Length(); j++) { + sepStorage[i][j] = static_cast( + jsSep.Get(j).As().Int32Value()); + } + + prompts[i] = jsPrompts.Get(i).As().Utf8Value(); + + Napi::Array jsImgs = jsBitmaps.Get(i).As(); + bitmapBytes[i].resize(jsImgs.Length()); + for (uint32_t j = 0; j < jsImgs.Length(); j++) { + Napi::Value v = jsImgs.Get(j); + // Uint8Array specifically, not any TypedArray: an Int32Array would + // pass a bare IsTypedArray() check and then be reinterpreted as raw + // bytes, silently feeding the decoder the wrong buffer. + const bool isU8 = v.IsTypedArray() && + v.As().TypedArrayType() == napi_uint8_array; + if (!v.IsBuffer() && !isU8) { + throw Napi::Error::New(env, + "_storePrefillMultimodal: bitmaps must be Buffer/Uint8Array"); + } + if (v.IsBuffer()) { + auto buf = v.As>(); + bitmapBytes[i][j].assign(buf.Data(), buf.Data() + buf.Length()); + } else { + auto ta = v.As(); + auto* base = static_cast(ta.ArrayBuffer().Data()) + ta.ByteOffset(); + bitmapBytes[i][j].assign(base, base + ta.ByteLength()); + } + } + } + + // A handle may appear at most once per call — the kernel's rule, applied + // to the marshaled handles: this path dispatches one branch at a time, so + // decode_scatter never sees the pair, and the values checked are exactly + // the values dispatched. + try { + lloyal::branch::require_distinct_handles(handles, "_storePrefillMultimodal"); + } catch (const std::exception& e) { + throw Napi::Error::New(env, e.what()); + } + + const int32_t nEmbdInp = llama_model_n_embd_inp(_model.get()); + + auto* worker = new StorePrefillMultimodalWorker( + env, _branchStore, _mtmdContext, nEmbdInp, + std::move(handles), std::move(sepStorage), std::move(prompts), + std::move(bitmapBytes)); + worker->Queue(); + return worker->GetPromise(); +} + +Napi::Value SessionContext::_cellsMultimodal(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (!_mtmdContext) { + throw Napi::Error::New(env, + "_cellsMultimodal requires a multimodal context — pass mmprojPath to createContext"); + } + if (info.Length() < 3 || !info[0].IsArray() || !info[1].IsString() || !info[2].IsArray()) { + throw Napi::Error::New(env, + "_cellsMultimodal requires (sepTokens[], prompt, bitmaps[])"); + } + + // One delta at a time: admission is a per-delta question, and SETTLE already + // walks its items in order deducting from a running headroom. + Napi::Array jsSep = info[0].As(); + std::vector sep(jsSep.Length()); + for (uint32_t i = 0; i < jsSep.Length(); i++) { + sep[i] = static_cast(jsSep.Get(i).As().Int32Value()); + } + + std::string prompt = info[1].As().Utf8Value(); + + // Marshal on the JS thread — the worker owns copies (Buffers must never be + // touched off-thread). + Napi::Array jsImgs = info[2].As(); + std::vector> bitmapBytes(jsImgs.Length()); + for (uint32_t j = 0; j < jsImgs.Length(); j++) { + Napi::Value v = jsImgs.Get(j); + // Uint8Array specifically, not any TypedArray: an Int32Array would pass a + // bare IsTypedArray() check and then be reinterpreted as raw bytes. + const bool isU8 = v.IsTypedArray() && + v.As().TypedArrayType() == napi_uint8_array; + if (!v.IsBuffer() && !isU8) { + throw Napi::Error::New(env, "_cellsMultimodal: bitmaps must be Buffer/Uint8Array"); + } + if (v.IsBuffer()) { + auto buf = v.As>(); + bitmapBytes[j].assign(buf.Data(), buf.Data() + buf.Length()); + } else { + auto ta = v.As(); + auto* base = static_cast(ta.ArrayBuffer().Data()) + ta.ByteOffset(); + bitmapBytes[j].assign(base, base + ta.ByteLength()); + } + } + + const int32_t nEmbdInp = llama_model_n_embd_inp(_model.get()); + + auto* worker = new CellsMultimodalWorker( + env, _mtmdContext, nEmbdInp, + std::move(sep), std::move(prompt), std::move(bitmapBytes)); + worker->Queue(); + return worker->GetPromise(); +} + +Napi::Value SessionContext::supportsVision(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + return Napi::Boolean::New(env, + _mtmdContext != nullptr && mtmd_support_vision(_mtmdContext)); +} + +Napi::Value SessionContext::supportsAudio(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + return Napi::Boolean::New(env, + _mtmdContext != nullptr && mtmd_support_audio(_mtmdContext)); +} + Napi::Value SessionContext::_storeMergeLogits(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); diff --git a/src/SessionContext.hpp b/src/SessionContext.hpp index ae6442b..ce002c9 100644 --- a/src/SessionContext.hpp +++ b/src/SessionContext.hpp @@ -11,6 +11,11 @@ #include #include +// Forward declaration at GLOBAL scope — mtmd_context is llama.cpp/tools/mtmd's +// opaque C type. Declaring it here (not inside the namespace below) keeps the +// member/param types referring to ::mtmd_context, matching . +struct mtmd_context; + namespace liblloyal_node { /** @@ -75,6 +80,14 @@ class SessionContext : public Napi::ObjectWrap { int32_t nBatch = lloyal::defaults::N_BATCH_INIT ); + /** + * Attach the mtmd (multimodal) context loaded from an mmproj file. + * Called by CreateContext when options.mmprojPath is set. Ownership + * transfers to this SessionContext (freed before the model in both + * the destructor and dispose()). + */ + void initializeMultimodal(struct mtmd_context* mtmd); + private: // ===== CORE PRIMITIVES ===== @@ -105,6 +118,7 @@ class SessionContext : public Napi::ObjectWrap { * Returns: string */ Napi::Value tokenToText(const Napi::CallbackInfo& info); + Napi::Value tokenToBytes(const Napi::CallbackInfo& info); /** * Check if token is a stop token (EOS) @@ -278,6 +292,25 @@ class SessionContext : public Napi::ObjectWrap { Napi::Value _storeCommit(const Napi::CallbackInfo& info); Napi::Value _storePrefill(const Napi::CallbackInfo& info); + + /** + * Multimodal prefill: per-branch sep tokens + templated prompt (with + * media markers) + image bytes. The walk itself is liblloyal's: + * `MtmdSource` yields TEXT/IMAGE segments in order and + * `BranchStore::decode_segments` places them (token rail / embedding rail). + * Args: (handles: number[], sepTokens: number[][], prompts: string[], + * bitmaps: Buffer[][]) + * Returns: Promise<{tokensDecoded, positionAdvance, error?, rc?, partial?}[]> + * — per-entry outcomes; a failed entry carries `error`, and when the + * failure came from llama_decode, its `rc` and `partial` (see DecodeError). + */ + Napi::Value _storePrefillMultimodal(const Napi::CallbackInfo& info); + Napi::Value _cellsMultimodal(const Napi::CallbackInfo& info); + + /** True when the loaded mmproj has a vision encoder (no mmproj → false). */ + Napi::Value supportsVision(const Napi::CallbackInfo& info); + /** True when the loaded mmproj has an audio encoder (no mmproj → false). */ + Napi::Value supportsAudio(const Napi::CallbackInfo& info); Napi::Value _storeMergeLogits(const Napi::CallbackInfo& info); Napi::Value _storeRetainOnly(const Napi::CallbackInfo& info); Napi::Value _storeAvailable(const Napi::CallbackInfo& info); @@ -292,6 +325,10 @@ class SessionContext : public Napi::ObjectWrap { std::shared_ptr _model; llama_context* _context = nullptr; + /// mtmd multimodal context (owned; nullptr when no mmproj was loaded). + /// Freed BEFORE _model in ~SessionContext and dispose() — mtmd holds a + /// reference to the model. + struct mtmd_context* _mtmdContext = nullptr; bool _disposed = false; int32_t _nBatch = lloyal::defaults::N_BATCH_INIT; diff --git a/src/index.ts b/src/index.ts index 95cf5c8..cc07c0d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -80,6 +80,80 @@ const tryLoadPackage = ( } }; +/** + * One native addon image per thread — a second one is refused, loudly. + * + * Node identifies a module by its REALPATH, so a symlinked package resolves + * its own dependencies from its own tree rather than the host project's. A + * locally linked `@lloyal-labs/rig` therefore reaches a DIFFERENT copy of + * `@lloyal-labs/lloyal.node` than the project does, and both get dlopen'd + * into one process. + * + * Each image carries its own copy of every C++ static — its own BranchStore, + * its own slot deque, its own freelist. A branch handle minted against one + * image and resolved against the other reads a garbage slot index, and the + * process dies with SIGSEGV or SIGBUS inside `allocate_slot`, code that has + * nothing to do with the mistake. This guard turns that into one startup + * error naming both packages. + * + * Identity is the test, not the path: Node caches a native addon by resolved + * filename, so two module instances requiring the SAME file share one exports + * object and are correct. `globalThis` is per-thread, so a worker thread that + * loads its own image is unaffected. + */ +const IMAGE_CLAIM = Symbol.for("@lloyal-labs/lloyal.node:image"); + +type ImageClaim = { binding: NativeBinding; owner: string }; + +/** + * Addon images already in this thread's require cache that are NOT ours. + * + * The symbol claim below only catches a sibling that also carries this guard. + * A copy old enough to predate it claims nothing, so scan the cache directly: + * a native addon is cached by resolved filename, and the entry whose exports + * ARE our binding is our own. Anything else named lloyal.node is a second + * image. This sees a foreign copy that loaded BEFORE us at any version; one + * that loads after us is caught only once it carries this guard too. + */ +const foreignImages = (binding: NativeBinding): string[] => { + const cache = (require as unknown as { + cache?: Record; + }).cache; + if (!cache) return []; + return Object.keys(cache).filter( + (file) => + path.basename(file) === "lloyal.node" && + cache[file]?.exports !== binding, + ); +}; + +const claimImage = (binding: NativeBinding): NativeBinding => { + const globals = globalThis as unknown as Record< + symbol, + ImageClaim | undefined + >; + const held = globals[IMAGE_CLAIM]; + const foreign = held ? [] : foreignImages(binding); + + if (!held && foreign.length === 0) { + globals[IMAGE_CLAIM] = { binding, owner: __dirname }; + return binding; + } + if (held?.binding === binding) return binding; + + throw new Error( + `[lloyal.node] Two different native addon images in one process.\n` + + ` already loaded: ${held?.owner ?? foreign.join(", ")}\n` + + ` now loading : ${__dirname}\n` + + `Each image has its own BranchStore and freelist, so a branch handle ` + + `that crosses between them faults in allocate_slot. This is almost ` + + `always a local dev link: a symlinked @lloyal-labs package resolves ` + + `lloyal.node from its own node_modules instead of the project's. Point ` + + `every copy at one build, or set LLOYAL_BACKEND_DIR to the directory ` + + `containing lloyal.node.`, + ); +}; + /** * Load native binary for a specific GPU variant * @@ -120,9 +194,12 @@ const tryLoadPackage = ( * failure throws and never falls through — a corrupt pack must not * silently degrade to the npm CPU package. * - * @param variant GPU variant: 'cuda', 'vulkan', or undefined for CPU + * @param variant GPU variant — `'cuda'`, `'vulkan'` or `'default'`; `undefined` + * resolves from `LLOYAL_GPU` and the platform (see {@link GpuVariant}) * @returns Native binary module with createContext method - * @throws Error if no binary available for the current platform + * @throws Error if no binary is available for the current platform, or if a + * DIFFERENT addon image is already loaded in this thread — see {@link + * claimImage} for why a second image is fatal rather than merely wasteful. * * @example * ```typescript @@ -138,7 +215,11 @@ const tryLoadPackage = ( * * @category Core */ -export const loadBinary = (variant?: GpuVariant): NativeBinding => { +export const loadBinary = (variant?: GpuVariant): NativeBinding => + claimImage(resolveBinary(variant)); + +/** The resolution chain documented on {@link loadBinary}. */ +const resolveBinary = (variant?: GpuVariant): NativeBinding => { const resolvedVariant = variant ?? process.env.LLOYAL_GPU; const noFallback = process.env.LLOYAL_NO_FALLBACK === "1"; const useLocal = process.env.LLOYAL_LOCAL === "1"; diff --git a/test/backend-pack-unit.ts b/test/backend-pack-unit.ts index 48423f2..aac0142 100644 --- a/test/backend-pack-unit.ts +++ b/test/backend-pack-unit.ts @@ -18,7 +18,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import * as zlib from 'node:zlib'; -import { sha256Hex, verifyPlatformSignature } from '../src/verify'; +import { sha256Hex, verifyPlatformSignature } from '../dist/verify.js'; import { backendPackCacheDir, detectCudaRuntime, @@ -29,7 +29,7 @@ import { resolveBackendPackDirSync, type BackendPackManifest, type CommandRunner, -} from '../src/backend-pack'; +} from '../dist/backend-pack.js'; let passed = 0; async function test(name: string, fn: () => void | Promise): Promise { diff --git a/test/fixtures/red-square-blue-circle.png b/test/fixtures/red-square-blue-circle.png new file mode 100644 index 0000000..ceb9297 Binary files /dev/null and b/test/fixtures/red-square-blue-circle.png differ diff --git a/test/integration.ts b/test/integration.ts index 82d80bb..891d208 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -740,6 +740,26 @@ async function testTokenizer(ctx: SessionContext): Promise { const eogText: string = ctx.tokenToText(eog); assert(eogText.length > 0, `EOS text: "${eogText}"`); + // tokenToBytes — the byte-level twin of tokenToText. A BPE piece can end + // mid-character: decoded one at a time the pieces tear into U+FFFD, while + // their bytes concatenate back to the text. (SentencePiece models keep the + // prefix space on the first piece that detokenize() strips.) + // Typed structurally, as the multimodal block does for `_storePrefill*`: + // the published sdk's SessionContext (the `^3` devDependency CI installs) + // predates this method, and the linked workspace hides that locally. + const bytes = ctx as unknown as { tokenToBytes(token: number): Uint8Array }; + const multibyte = '日本語 🎉 𠜎𠜱 👨‍👩‍👧‍👦'; + const mbTokens: number[] = await ctx.tokenize(multibyte, false); + const pieces: Uint8Array[] = mbTokens.map((t: number) => bytes.tokenToBytes(t)); + assert(pieces.every((p: Uint8Array) => Buffer.isBuffer(p)), + `tokenToBytes returns a Buffer per token (${pieces.length} pieces)`); + const torn: number = pieces.filter((p: Uint8Array) => Buffer.from(p).toString('utf8').includes('\uFFFD')).length; + assert(torn > 0, `${torn} of ${pieces.length} pieces end mid-character`); + const joined: string = Buffer.concat(pieces).toString('utf8'); + const detok: string = await ctx.detokenize(mbTokens); + assert(joined === detok || joined === ' ' + detok, + `bytes concatenate to detokenize(): ${JSON.stringify(joined)}`); + // tokenize with addSpecial const withSpecial: number[] = await ctx.tokenize('Hello world', true); const noSpecial: number[] = await ctx.tokenize('Hello world', false); @@ -2237,6 +2257,482 @@ async function testRerankConcurrent(): Promise { // MAIN // ═══════════════════════════════════════════════════════════════════════════ +// ═══════════════════════════════════════════════════════════════════════════ +// MULTIMODAL (mtmd) TESTS — the embedding rail +// ═══════════════════════════════════════════════════════════════════════════ +// +// Gated on a VL model + mmproj pair being present. Tier selection: +// env LLAMA_VL_MODEL + LLAMA_VL_MMPROJ (explicit), else +// Qwen3.5-4B + its mmproj (local dev — M-RoPE, content assertions), else +// SmolVLM-256M + its mmproj (CI tier — plain positions, mechanics only). + +const VL_PAIRS: Array<{ model: string; mmproj: string; mrope: boolean; strict: boolean }> = [ + { + model: path.join(__dirname, '../models/Qwen3.5-4B-Q4_K_M.gguf'), + mmproj: path.join(__dirname, '../models/mmproj-Qwen3.5-4B-F16.gguf'), + mrope: true, + strict: true, + }, + { + model: path.join(__dirname, '../models/SmolVLM-256M-Instruct-Q8_0.gguf'), + mmproj: path.join(__dirname, '../models/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf'), + mrope: false, + strict: false, + }, +]; + +function pickVlPair(): { model: string; mmproj: string; mrope: boolean; strict: boolean } | null { + if (process.env.LLAMA_VL_MODEL && process.env.LLAMA_VL_MMPROJ) { + const model = path.resolve(process.env.LLAMA_VL_MODEL); + return { + model, + mmproj: path.resolve(process.env.LLAMA_VL_MMPROJ), + mrope: /qwen/i.test(model), + strict: /qwen3\.5-4b|qwen3\.8/i.test(model), + }; + } + for (const p of VL_PAIRS) { + if (fs.existsSync(p.model) && fs.existsSync(p.mmproj)) return p; + } + return null; +} + +const MEDIA_MARKER = '<__media__>'; + +async function testMultimodal(): Promise { + const vl = pickVlPair(); + if (!vl) { + console.log('\n--- Multimodal: SKIPPED (no VL model + mmproj pair in models/) ---'); + return; + } + console.log(`\n--- Multimodal (${path.basename(vl.model)}, mrope=${vl.mrope}) ---`); + + const IMG: Buffer = fs.readFileSync(path.join(__dirname, 'fixtures/red-square-blue-circle.png')); + + // q4_0 KV (assertion 1) + small nBatch so any real image sub-chunks + // (assertion 3 — decode::embd's internal loop is the MAIN path). + const ctx: SessionContext = await addon.createContext({ + modelPath: vl.model, + mmprojPath: vl.mmproj, + nCtx: 8192, + nBatch: 256, + nSeqMax: 8, + typeK: 'q4_0', + typeV: 'q4_0', + nThreads: 4, + } as never); + const mm = ctx as unknown as { + _storePrefillMultimodal( + handles: number[], seps: number[][], prompts: string[], bitmaps: Buffer[][], + ): Promise>; + _storePrefill(handles: number[], tokenArrays: number[][]): Promise; + _storeKvPressure(): { cellsUsed: number }; + supportsVision(): boolean; + supportsAudio(): boolean; + }; + + try { + // Probes + assert(mm.supportsVision() === true, 'supportsVision() → true with mmproj loaded'); + assert(mm.supportsAudio() === false, 'supportsAudio() → false (vision-only mmproj)'); + + // Assertion 2a — marker survives the SYSTEM-ONLY template route (the + // spine-header path; on Qwen3.5 this exercises the sentinel-retry). + const { prompt: sysPrompt } = await ctx.formatChat( + JSON.stringify([{ + role: 'system', + content: [ + { type: 'text', text: 'You can see the attached reference image.' }, + { type: 'media_marker', text: MEDIA_MARKER }, + ], + }]), + { addGenerationPrompt: false } as never, + ); + assert(sysPrompt.includes(MEDIA_MARKER), + 'marker verbatim through system-only route (media_marker content part)'); + + // The main user-turn prompt (generation prompt on — we produce after) + const { prompt: userPrompt } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: 'You are a vision assistant. Answer briefly.' }, + { + role: 'user', + content: [ + { type: 'text', text: 'What shapes and colors are in this image?' }, + { type: 'media_marker', text: MEDIA_MARKER }, + ], + }, + ]), { enableThinking: false } as never); + assert(userPrompt.includes(MEDIA_MARKER), 'marker verbatim through user-turn route'); + + // Assertion 1 + 4 — multimodal prefill on q4_0 KV; cells vs position + const root = Branch.create(ctx, 0, { temperature: 0 }); + const cells0: number = mm._storeKvPressure().cellsUsed; + const res = await mm._storePrefillMultimodal([root.handle], [[]], [userPrompt], [[IMG]]); + const { tokensDecoded, positionAdvance } = res[0]; + assert(tokensDecoded > 0, `prefill decoded ${tokensDecoded} tokens`); + assert(root.position === positionAdvance, + `branch position (${root.position}) == positionAdvance (${positionAdvance})`); + const cells1: number = mm._storeKvPressure().cellsUsed; + assert(cells1 - cells0 === tokensDecoded, + `cells grew by tokensDecoded (${cells1 - cells0} == ${tokensDecoded})`); + if (vl.mrope) { + assert(positionAdvance < tokensDecoded, + `M-RoPE decouple: positionAdvance (${positionAdvance}) < tokensDecoded (${tokensDecoded})`); + } else { + assert(positionAdvance === tokensDecoded, + `plain positions: positionAdvance == tokensDecoded (${tokensDecoded})`); + } + // Assertion 3 — with nBatch 256, a real image (>=256 rows on these + // models) exercised decode::embd's sub-chunk loop; reaching here with a + // coherent KV is the observable (a broken view/pos repack would throw or + // corrupt the grounded answer below). + + // Grounded answer on the image + const gen: number[] = []; + for (let i = 0; i < 48; i++) { + const { token, isStop } = await root.produce(); + if (isStop) break; + await root.commit(token); + gen.push(token); + } + assert(gen.length > 0, `generated ${gen.length} tokens after image prefill`); + const answer: string = await ctx.detokenize(gen); + if (vl.strict) { + assert(/red|square|blue|circle/i.test(answer), + `grounded answer names the content: "${answer.trim()}"`); + } else { + ok(`answer (mechanics tier): "${answer.trim()}"`); + } + const cellsAfterGen: number = mm._storeKvPressure().cellsUsed; + + // Assertion 5 — spine-share: fork ×2 AFTER the image; forks add no cells + const childA = await root.fork(); + const childB = await root.fork(); + assert(mm._storeKvPressure().cellsUsed === cellsAfterGen, + 'fork ×2 adds zero cells (image KV shared, not copied)'); + + const sep: number[] = ctx.getTurnSeparator(); + const { prompt: qPrompt } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: '' }, + { role: 'user', content: 'In one word, what color is the square?' }, + ]), { enableThinking: false } as never); + const qToks: number[] = await ctx.tokenize(qPrompt, false); + const childToks: number[] = [...sep, ...qToks]; + + let aCells = 0; + for (const child of [childA, childB]) { + const before: number = mm._storeKvPressure().cellsUsed; + await child.prefill(childToks); + const cgen: number[] = []; + for (let i = 0; i < 8; i++) { + const { token, isStop } = await child.produce(); + if (isStop) break; + await child.commit(token); + cgen.push(token); + } + assert(cgen.length > 0, 'forked child answers over the shared image'); + if (vl.strict && child === childA) { + const ctext: string = await ctx.detokenize(cgen); + assert(/red/i.test(ctext), `child sees the image (square color): "${ctext.trim()}"`); + } + if (child === childA) aCells = mm._storeKvPressure().cellsUsed - before; + } + + // Assertion 4b — releasing a fork recovers exactly its OWN cells (text + // suffix only, never the shared image): the release-slack fix. + const beforeRelease: number = mm._storeKvPressure().cellsUsed; + await childA.prune(); + const released: number = beforeRelease - mm._storeKvPressure().cellsUsed; + assert(released === aCells, + `release recovered exactly the fork's own cells (${released} == ${aCells})`); + await childB.prune(); + + // Assertion 6 — prefill ending AT the marker (no trailing template + // text): logits must be available for an immediate produce(). + const term = Branch.create(ctx, 0, { temperature: 0 }); + const termRes = await mm._storePrefillMultimodal( + [term.handle], [[]], [`Describe: ${MEDIA_MARKER}`], [[IMG]]); + assert(termRes[0].tokensDecoded > 0, 'marker-terminal prefill decoded'); + const t1 = await term.produce(); + assert(Number.isInteger(t1.token), 'produce() works immediately after marker-terminal prefill'); + await term.prune(); + + // Assertion 7 — duplicate handle in one store call throws (fail loud) + let dupThrew = false; + try { + await mm._storePrefill([root.handle, root.handle], [[qToks[0]], [qToks[0]]]); + } catch { dupThrew = true; } + assert(dupThrew, 'duplicate handle in one _storePrefill rejects'); + + // The multimodal cohort coerces handles with Uint32Value: 5 and 5.5 name + // the SAME branch, and its duplicate guard must see that too. + let fracDupThrew = false; + try { + await mm._storePrefillMultimodal( + [root.handle, root.handle + 0.5], [[], []], [userPrompt, userPrompt], [[IMG], [IMG]]); + } catch { fracDupThrew = true; } + assert(fracDupThrew, 'fractional duplicate handle in one _storePrefillMultimodal rejects'); + + // Marker/bitmap count mismatch is reported PER ENTRY, not by rejecting the + // whole call. A cohort carries N independent branches: rejecting would lose + // which of them landed, and six agents settling images must not lose five + // because one page was corrupt. `MtmdSource` still refuses in its + // CONSTRUCTOR — before any decode, branch untouched — so the entry is + // rejected just as hard; only the reporting channel changed. + let mismatchErr = ''; + { + const b = Branch.create(ctx, 0, { temperature: 0 }); + try { + const r = await mm._storePrefillMultimodal( + [b.handle], [[]], [`x ${MEDIA_MARKER}`], [[IMG, IMG]]); + mismatchErr = r[0].error ?? ''; + assert(r[0].tokensDecoded === 0, + 'a refused entry decodes nothing (its branch is untouched)'); + } finally { await b.prune(); } + } + assert(/marker count/i.test(mismatchErr), + `marker/bitmap mismatch reported per entry: "${mismatchErr}"`); + + // Assertion 8 — frames (the video-carrying contract): several marker'd + // frames + timestamp text in ONE prefill. + const frames = Branch.create(ctx, 0, { temperature: 0 }); + const framesPrompt = + `Video frames. t=0s: ${MEDIA_MARKER} t=1s: ${MEDIA_MARKER} t=2s: ${MEDIA_MARKER} ` + + `What color is the square across the frames? Answer in one word:`; + const fRes = await mm._storePrefillMultimodal( + [frames.handle], [[]], [framesPrompt], [[IMG, IMG, IMG]]); + assert(fRes[0].tokensDecoded > 0 && fRes[0].positionAdvance > 0, + `frames prefill decoded ${fRes[0].tokensDecoded} tokens (${fRes[0].positionAdvance} positions)`); + const fgen: number[] = []; + for (let i = 0; i < 8; i++) { + const { token, isStop } = await frames.produce(); + if (isStop) break; + await frames.commit(token); + fgen.push(token); + } + assert(fgen.length > 0, 'answer over multi-frame prefill'); + if (vl.strict) { + const ftext: string = await ctx.detokenize(fgen); + assert(/red/i.test(ftext), `temporal answer grounded: "${ftext.trim()}"`); + } + await frames.prune(); + + // Assertion 9 — batched fan-out over the shared image, through the PUBLIC + // BranchStore surface. The spine-share above proves forks are free and + // drives its children ONE AT A TIME; this drives N children TOGETHER, each + // asking a different question, at one dispatch per tick. That is the + // README's "N branches, 1 GPU call" over a prefix encoded exactly once — + // and it exercises the JS→N-API marshalling of handle/token arrays that + // the kernel-level test cannot reach. + const store = new BranchStore(ctx); + // Short SENTENCES, not single words: a one-word answer emits its token and + // stops, so the loop below would do a single dispatch and prove nothing + // about sustained batching. Several tokens each keeps all four branches + // live across multiple ticks, which is the property under test. + const asks: Array<{ q: string; expect: RegExp }> = [ + { q: 'Describe the red object in a short sentence.', expect: /square/i }, + { q: 'Describe the blue object in a short sentence.', expect: /circle/i }, + { q: 'In a short sentence, what color is the square?', expect: /red/i }, + { q: 'In a short sentence, what color is the circle?', expect: /blue/i }, + ]; + + const cellsBeforeFan: number = mm._storeKvPressure().cellsUsed; + const kids: InstanceType[] = []; + for (let i = 0; i < asks.length; i++) kids.push(await root.fork()); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan, + `fan-out: fork ×${asks.length} over the image adds zero cells`); + + // Every child's question in ONE variable-length scatter. + const suffixes: number[][] = []; + for (const a of asks) { + const { prompt: p } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: '' }, + { role: 'user', content: a.q }, + ]), { enableThinking: false } as never); + suffixes.push([...sep, ...(await ctx.tokenize(p, false))]); + } + const suffixTotal: number = suffixes.reduce((n, s) => n + s.length, 0); + await store.prefill(kids.map((b, i) => [b, suffixes[i]] as [typeof b, number[]])); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan + suffixTotal, + `fan-out: costs only the text suffixes (${suffixTotal}), never the image again`); + + // One store.commit() per tick carries every live child. + const fanToks: number[][] = asks.map(() => []); + const widths: number[] = []; // branches carried by each dispatch + for (let step = 0; step < 16; step++) { + const entries: Array<[InstanceType, number]> = []; + for (let i = 0; i < kids.length; i++) { + if (fanToks[i].length && fanToks[i][fanToks[i].length - 1] === -1) continue; + const { token, isStop } = await kids[i].produce(); + if (isStop) { fanToks[i].push(-1); continue; } + fanToks[i].push(token); + entries.push([kids[i], token]); + } + if (!entries.length) break; + await store.commit(entries); + widths.push(entries.length); + } + // Sustained batching is several dispatches that each carry SEVERAL + // branches. Counting non-empty ticks would pass when three children stop + // at tick one and a lone survivor runs on — serial decoding, not batching. + const batched: number = widths.filter((w) => w > 1).length; + assert(batched > 1, + `fan-out: dispatch widths ${widths.join(',')} — ${batched} carried several branches`); + + for (let i = 0; i < asks.length; i++) { + const toks: number[] = fanToks[i].filter((t) => t !== -1); + assert(toks.length > 0, `fan-out: child ${i} produced an answer`); + const text: string = await ctx.detokenize(toks); + if (vl.strict) { + assert(asks[i].expect.test(text), + `fan-out: "${asks[i].q}" → "${text.trim()}"`); + } else { + ok(`fan-out child ${i} (mechanics tier): "${text.trim()}"`); + } + } + + for (const k of kids) await k.prune(); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan, + 'fan-out: children release their own cells, image prefix intact'); + + await root.prune(); + } finally { + ctx.dispose(); + } +} + +// ============================================================================ +// A failed decode says what landed — the JavaScript side of the kernel's +// DecodeError. The kernel's own suite proves the producers; this proves the +// last hop: `rc` and `partial` arrive on the rejection (token rail) and on +// the per-entry result (media rail), and the books a caller reads agree. +// Runs on the default model and, when present, on the production default +// (Qwen3.5-4B: a Gated DeltaNet hybrid, the carrier that cannot be rewound — +// the reason the contract is prune-and-replay in the first place). +// ============================================================================ +async function testDecodeFailure(): Promise { + console.log('\n--- Decode failure: partial ---'); + const failureOf = (err: unknown): { rc?: number; partial?: boolean; message?: string } => + err as { rc?: number; partial?: boolean; message?: string }; + const pressure = (c: SessionContext): { cellsUsed: number } => + (c as unknown as { _storeKvPressure(): { cellsUsed: number } })._storeKvPressure(); + const filler = (n: number, base: number, vocab: number): number[] => + Array.from({ length: n }, (_, j) => (base + j) % vocab); + + const models: string[] = [MODEL_PATH]; + const gdn = path.join(__dirname, '../models/Qwen3.5-4B-Q4_K_M.gguf'); + if (path.resolve(gdn) !== path.resolve(MODEL_PATH) && fs.existsSync(gdn)) models.push(gdn); + + for (const modelPath of models) { + console.log(` model: ${path.basename(modelPath)}`); + + // A later scatter chunk finds no KV slot: rc 1, partial true, the landed + // children moved, the refused one did not and still works. + { + const ctx: SessionContext = await addon.createContext({ modelPath, nCtx: 256, nBatch: 128, nSeqMax: 4, nThreads: 4 }); + try { + const vocab = ctx.vocabSize; + const prompt = await ctx.tokenize('Hello'); + assert(prompt.length > 0 && prompt.length < 40, 'scatter: a short prefix'); + const store = new BranchStore(ctx); + const root = Branch.create(ctx, 0, { temperature: 0 }); + await root.prefill(prompt); + const p = root.position; + const cells0 = pressure(ctx).cellsUsed; + const kids = [await root.fork(), await root.fork(), await root.fork()]; + // 100 tokens each under nBatch 128: three chunks. 256 cells hold p+200, not p+300. + let caught: ReturnType | null = null; + try { + await store.prefill([ + [kids[0], filler(100, 1000, vocab)], + [kids[1], filler(100, 2000, vocab)], + [kids[2], filler(100, 3000, vocab)], + ]); + } catch (err) { caught = failureOf(err); } + assert(caught !== null, 'scatter: the third chunk is refused'); + assert(caught!.rc === 1, `scatter: rc 1 on the rejection (got ${caught!.rc})`); + assert(caught!.partial === true, `scatter: partial:true on the rejection (got ${caught!.partial})`); + assert(kids[0].position === p + 100 && kids[1].position === p + 100, 'scatter: landed children moved'); + assert(kids[2].position === p, 'scatter: the refused child did not move'); + assert(pressure(ctx).cellsUsed === cells0 + 200, 'scatter: landed cells charged, refused cells not'); + await store.prefill([[kids[2], filler(10, 4000, vocab)]]); + assert(kids[2].position === p + 10, 'scatter: the intact child still prefills'); + await root.pruneSubtree(); + assert(pressure(ctx).cellsUsed === 0, 'scatter: the pool is whole after prune'); + ok('scatter: rc 1 + partial:true in JS; landed 2/3, intact child prefilled, pool reclaimed'); + } finally { + ctx.dispose(); + } + } + + // A chunked prefill that dies mid-way: poisoned, says so, prune reclaims it. + { + const ctx: SessionContext = await addon.createContext({ modelPath, nCtx: 256, nBatch: 64, nSeqMax: 2, nThreads: 4 }); + try { + const vocab = ctx.vocabSize; + const b = Branch.create(ctx, 0, { temperature: 0 }); + let caught: ReturnType | null = null; + try { await b.prefill(filler(300, 1000, vocab)); } catch (err) { caught = failureOf(err); } + assert(caught !== null && caught.rc === 1 && caught.partial === true, + `prefill: rc 1 + partial:true (got rc=${caught?.rc} partial=${caught?.partial})`); + assert(b.position === 0, 'prefill: the books did not move'); + assert(pressure(ctx).cellsUsed === 0, 'prefill: nothing charged'); + await b.prune(); + const fresh = Branch.create(ctx, 0, { temperature: 0 }); + await fresh.prefill(filler(10, 5000, vocab)); + assert(fresh.position === 10 && pressure(ctx).cellsUsed === 10, 'prefill: prune reclaimed the orphans — a fresh branch prefills'); + await fresh.prune(); + ok('prefill: poisoned reported as partial:true, reclaimed by prune'); + } finally { + ctx.dispose(); + } + } + } + + // The media rail reports PER ENTRY: an image whose cells outrun the pool + // comes back with error + rc 1 + partial:true, its siblings untouched. + const vl = pickVlPair(); + const bigImage = path.join(__dirname, '../liblloyal/tests/fixtures/cat.jpg'); + if (!vl || !fs.existsSync(bigImage)) { + console.log(' [SKIP] media: no VL pair or no large fixture'); + return; + } + { + const ctx: SessionContext = await addon.createContext({ + modelPath: vl.model, mmprojPath: vl.mmproj, nCtx: 256, nBatch: 16, nSeqMax: 2, nThreads: 4, + } as never); + try { + const mm = ctx as unknown as { + _storePrefillMultimodal(handles: number[], seps: number[][], prompts: string[], bitmaps: Buffer[][]): + Promise>; + _cellsMultimodal(sep: number[], prompt: string, bitmaps: Buffer[]): Promise; + }; + const IMG: Buffer = fs.readFileSync(bigImage); + const { prompt } = await ctx.formatChat(JSON.stringify([ + { role: 'user', content: [{ type: 'text', text: 'Describe this image.' }, { type: 'media_marker', text: MEDIA_MARKER }] }, + ])); + const cells = await mm._cellsMultimodal([], prompt, [IMG]); + if (cells <= 256) { + console.log(` [SKIP] media: the image costs ${cells} cells and fits in 256`); + } else { + const b = Branch.create(ctx, 0, { temperature: 0 }); + const [r] = await mm._storePrefillMultimodal([b.handle], [[]], [prompt], [[IMG]]); + assert(typeof r.error === 'string', 'media: the entry failed'); + assert(r.rc === 1 && r.partial === true, + `media: entry carries rc 1 + partial:true (got rc=${r.rc} partial=${r.partial}: ${r.error})`); + await b.prune(); + assert(pressure(ctx).cellsUsed === 0, 'media: prune reclaimed the poisoned branch'); + ok(`media: ${cells} cells against a 256-cell pool → per-entry rc 1 + partial:true, reclaimed`); + } + } finally { + ctx.dispose(); + } + } +} + async function main(): Promise { let mainCtx: SessionContext | null = null; @@ -2278,6 +2774,8 @@ async function main(): Promise { await testSetSamplerParams(); await testSetGrammar(); await testBranchMetrics(); + await testMultimodal(); + await testDecodeFailure(); await testRerank(); await testRerankLargeCorpus(); await testRerankConcurrent(); diff --git a/test/matrix.json b/test/matrix.json index ebb68be..db0ff01 100644 --- a/test/matrix.json +++ b/test/matrix.json @@ -54,6 +54,18 @@ "file": "slim-ner-tool.gguf", "url": "https://huggingface.co/llmware/slim-ner-tool/resolve/main/slim-ner-tool.gguf", "usedBy": ["entities"] + }, + { + "name": "SmolVLM-256M", + "file": "SmolVLM-256M-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM-256M-Instruct-GGUF/resolve/main/SmolVLM-256M-Instruct-Q8_0.gguf", + "usedBy": ["multimodal"] + }, + { + "name": "SmolVLM-256M-mmproj", + "file": "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM-256M-Instruct-GGUF/resolve/main/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "usedBy": ["multimodal"] } ] } diff --git a/test/peers-unit.ts b/test/peers-unit.ts new file mode 100644 index 0000000..7d47c66 --- /dev/null +++ b/test/peers-unit.ts @@ -0,0 +1,38 @@ +/** + * The binding's peers must admit the set it ships beside — no GPU, no network. + * tsx + node:assert (the repo's test convention; run via `npm run test:unit`). + * + * dist/index.js re-exports values from @lloyal-labs/sdk and @lloyal-labs/lloyal-agents, + * so both are real peers. A plain range (`>=3.0.0`) never admits a prerelease: + * semver matches a prerelease only when a comparator names that exact + * major.minor.patch with a prerelease tag. So the alpha set (sdk 4.0.0-alpha.N, + * agents 6.0.0-alpha.N) failed every `npm install` that pinned it beside this + * binding — the scaffold's own install included — until the ranges named the + * tuples. Checked with the semver library npm resolves with. + */ +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { satisfies } from 'semver'; + +// CommonJS on purpose: tsconfig.test.json compiles the tests as CommonJS, so +// `__dirname`, not `import.meta.url` — the compile gate, not tsx, is the judge. +const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8')) as { + peerDependencies: Record; +}; +const peers = pkg.peerDependencies; + +/** Each peer: the stable a user has today, the prerelease this arc ships, the stable it becomes. */ +const ADMITS: Record = { + '@lloyal-labs/sdk': ['3.1.0', '4.0.0-alpha.3', '4.0.0'], + '@lloyal-labs/lloyal-agents': ['5.5.1', '6.0.0-alpha.3', '6.0.0'], +}; +for (const [name, versions] of Object.entries(ADMITS)) { + assert.ok(peers[name], `${name} is declared as a peer`); + for (const v of versions) { + assert.ok(satisfies(v, peers[name]), `${name} "${peers[name]}" admits ${v}`); + } +} +// The trap, stated: a plain range excludes the prerelease this set is made of. +assert.equal(satisfies('6.0.0-alpha.3', '>=3.0.0'), false); +console.log('peers-unit: ok');