From e1fb5dcb859d970243b1022ccd1585398295aa86 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 09:11:46 +1000 Subject: [PATCH 01/54] =?UTF-8?q?feat(mtmd):=20multimodal=20prefill=20?= =?UTF-8?q?=E2=80=94=20mmproj=20load,=20embedding=20rail,=20two-tier=20tes?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread image+text inference through the native layer. mtmd (llama.cpp's multimodal library) is consumed only by the binding worker; liblloyal stays mtmd-free (see liblloyal 42e5951 for the decode::embd / prefill_embd substrate). Build: - add_subdirectory(tools/mtmd) with EXCLUDE_FROM_ALL (skips its CLI targets) + LLAMA_INSTALL_VERSION set locally (directory-scoped inside llama.cpp, empty here otherwise) + MTMD_VIDEO OFF (compile-def only). - mtmd joins target_link_libraries (static into the .node on Windows) and the POST_BUILD SONAME foreach — platform packages and the DL pack pick libmtmd up through the existing globs, zero script changes. Binding: - createContext: mmprojPath (fail-loud — a configured mmproj that cannot load never silently degrades to text-only), imageMinTokens / imageMaxTokens pass-through, mtmd freed before the model in dtor+dispose. - _storePrefillMultimodal(handles, sepTokens, prompts, bitmaps) → per-branch {tokensDecoded, positionAdvance}. The worker owns prompt strings + copied image bytes and walks mtmd_tokenize chunks in order: TEXT → decode_scatter (token rail), IMAGE → encode → prefill_embd (embedding rail, section-major M-RoPE positions), AUDIO → explicit error. Tokenize flags {add_special: false, parse_special: true} match the text path exactly (no mid-conversation BOS around an image). - supportsVision() / supportsAudio() probes. Tests (two tiers, all assertions green): - Qwen3.5-4B + mmproj (M-RoPE): grounded answer on q4_0 KV; cells vs position decouple (233 cells / 51 positions); fork ×2 adds zero cells and both children answer over the shared image; release recovers exact cell counts; marker-terminal prefill; duplicate-handle and marker/bitmap-mismatch rejections; 3-frame timestamped prefill with a grounded temporal answer (the video-carrying contract). - SmolVLM-256M (plain positions, CI tier): same mechanics, positionAdvance == tokensDecoded. - matrix.json: SmolVLM pair as multimodal sidecars; models cache key → v2 (actions/cache never re-saves on hit). - test/fixtures/red-square-blue-circle.png: synthetic, assertable content. --- .github/workflows/release.yml | 4 +- CMakeLists.txt | 23 +- README.md | 49 ++- liblloyal | 2 +- src/SessionContext.cpp | 363 +++++++++++++++++++++++ src/SessionContext.hpp | 32 ++ test/fixtures/red-square-blue-circle.png | Bin 0 -> 2047 bytes test/integration.ts | 247 +++++++++++++++ test/matrix.json | 12 + 9 files changed, 727 insertions(+), 5 deletions(-) create mode 100644 test/fixtures/red-square-blue-circle.png diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ef99c1..65260c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -210,7 +210,9 @@ jobs: uses: actions/cache@v4 with: path: models/ - key: test-models-v1 + # v2: + SmolVLM-256M pair (multimodal CI tier). actions/cache never + # re-saves on a hit — bump the key when matrix.json gains files. + key: test-models-v2 - name: Download test models if: matrix.cross_compile != true diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c785f4..33479fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,6 +130,26 @@ if(MSVC) endif() endif() +# ============================================================================= +# mtmd (multimodal: clip/projector encode + media tokenization) +# ============================================================================= +# tools/mtmd is not built by llama.cpp under add_subdirectory (LLAMA_BUILD_TOOLS +# is OFF for non-standalone builds), so add the library directly. +# - EXCLUDE_FROM_ALL: mtmd's CMakeLists also declares CLI targets +# (llama-mtmd-cli etc.); only the linked `mtmd` library gets built. +# - LLAMA_INSTALL_VERSION is a normal variable scoped inside llama.cpp's +# directory — empty here, and mtmd's set_target_properties(VERSION ...) +# would receive "". Provide it before the add. +# - MTMD_VIDEO is a compile definition only (video decoding shells out to an +# ffmpeg binary at runtime); keep it OFF for a deterministic surface. +set(LLAMA_INSTALL_VERSION 0.0.0) +set(MTMD_VIDEO OFF CACHE BOOL "mtmd video support (runtime ffmpeg)" FORCE) +add_subdirectory(${LLAMA_CPP_DIR}/tools/mtmd mtmd EXCLUDE_FROM_ALL) +if(MSVC AND TARGET mtmd) + # Same u8"" literal workaround as the llama.cpp targets above. + target_compile_options(mtmd PRIVATE /Zc:char8_t-) +endif() + # Build liblloyal (INTERFACE library, links llama transitively) # This also sets up the llama/llama.h include structure automatically add_subdirectory(${LIBLLOYAL_DIR} liblloyal) @@ -202,6 +222,7 @@ endif() target_link_libraries(${PROJECT_NAME} PRIVATE liblloyal::liblloyal ${LLAMA_COMMON_TARGET} + mtmd md4c ${CMAKE_JS_LIB} ) @@ -288,7 +309,7 @@ if(BUILD_SHARED_LIBS) # libggml-cuda alone). Symlinks are intentionally avoided: they require admin / # Developer Mode on Windows (see liblloyal/CMakeLists.txt). This block is # Linux/macOS only (Windows links statically; SONAMEs don't apply to DLLs). - foreach(_lib IN ITEMS llama ${LLAMA_COMMON_TARGET} ggml ggml-base ggml-cpu ggml-metal ggml-cuda ggml-vulkan ggml-blas) + foreach(_lib IN ITEMS llama ${LLAMA_COMMON_TARGET} mtmd ggml ggml-base ggml-cpu ggml-metal ggml-cuda ggml-vulkan ggml-blas) if(TARGET ${_lib}) get_target_property(_lloyal_lib_type ${_lib} TYPE) if(_lloyal_lib_type STREQUAL "SHARED_LIBRARY") diff --git a/README.md b/README.md index 6f7e68e..974699e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GPU Tests](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml/badge.svg)](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml) [![npm](https://img.shields.io/npm/v/@lloyal-labs/lloyal.node.svg)](https://www.npmjs.com/package/@lloyal-labs/lloyal.node) [![License](https://img.shields.io/badge/license-FSL--1.1--Apache--2.0-blue.svg)](LICENSE) -[![llama.cpp](https://img.shields.io/badge/llama.cpp-b8795-green.svg)](https://github.com/ggml-org/llama.cpp/releases/tag/b8795) +[![llama.cpp](https://img.shields.io/badge/llama.cpp-b9581-green.svg)](https://github.com/ggml-org/llama.cpp/releases/tag/b9581) **Native backend for the lloyal inference platform.** @@ -117,11 +117,53 @@ const tokens = await ctx.tokenize("Hello world"); const sep = await ctx.getTurnSeparator(); ``` +## Multimodal (Vision) + +Load a model's multimodal projector (`mmproj` GGUF) alongside it and prefill +images into any branch's KV. Works with any llama.cpp-supported VL model — +the projector decides the position mode (M-RoPE for Qwen, plain for +llava-style) at runtime. + +```javascript +const ctx = await createContext({ + modelPath: "./Qwen3.5-4B-Q4_K_M.gguf", + mmprojPath: "./mmproj-F16.gguf", + nSeqMax: 8, +}); +ctx.supportsVision(); // true + +// One <__media__> marker per image, injected as a media_marker content part +const { prompt } = await ctx.formatChat(JSON.stringify([ + { role: "user", content: [ + { type: "text", text: "What is in this image?" }, + { type: "media_marker", text: "<__media__>" }, + ]}, +])); + +const bytes = fs.readFileSync("./photo.jpg"); // jpg/png/bmp/gif +const [{ tokensDecoded, positionAdvance }] = + await ctx._storePrefillMultimodal([handle], [[]], [prompt], [[bytes]]); +// ...then produce/commit as usual — the branch attends the image +``` + +The image lands in the KV as a shared prefix: fork the branch and every +child attends the image with zero re-encode. Several markers with several +images in one prefill also works — frames of a video, each preceded by a +timestamp, are just that. + +Options: `imageMinTokens` / `imageMaxTokens` cap per-image token budgets +(default: model metadata). A configured `mmprojPath` that fails to load +throws at `createContext` — never a silent fall back to text-only. Audio +bytes are rejected explicitly (no audio surface yet). + ## What This Package Provides **Native-only** (not in SDK): -- `createContext(options)` — load a GGUF model, return a `SessionContext` +- `createContext(options)` — load a GGUF model, return a `SessionContext`; + `mmprojPath` loads the multimodal projector beside it +- `_storePrefillMultimodal(...)` — image+text prefill into a branch's KV + (the embedding rail); `supportsVision()` / `supportsAudio()` probes - `loadBinary(options?)` — explicit GPU variant selection with automatic fallback - Prebuilt binaries for 13 platform/GPU combinations @@ -179,6 +221,9 @@ Integration tests run real inference across architectures: | SmolLM | SmolLM2 1.7B | chatml | | Ministral | Ministral 3B | mistral | +Multimodal tests run on two tiers: SmolVLM-256M (plain positions, CI) and +Qwen3.5-4B + mmproj (M-RoPE, local/GPU rig). + See [distribution.md](docs/distribution.md) for details. ## Ecosystem diff --git a/liblloyal b/liblloyal index a3558a0..42e5951 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit a3558a0619309ca41b7e044bef7329f653273527 +Subproject commit 42e5951ef26625efbbb51cb95a365838fd78d2ae diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index cdc4f22..d231ae7 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -674,6 +676,206 @@ class StorePrefillWorker : public Napi::AsyncWorker { std::vector> _tokenStorage; }; +/** + * AsyncWorker for multimodal prefill (the embedding rail beside the token + * rail). Owns everything it touches off-thread: per-branch sep tokens, + * templated prompt strings (containing <__media__> markers), and copied + * image bytes. All mtmd calls live HERE — liblloyal stays mtmd-free. + * + * Per branch, in order (segments are sequential store calls — start_pos is + * read from branch position at dispatch time): + * 1. sep tokens → decode_scatter (token rail) + * 2. mtmd_tokenize → interleaved TEXT/IMAGE chunks + * (add_special=false, parse_special=true — parity with the text path) + * 3. TEXT chunk → decode_scatter (ready token ids; never + * re-tokenized) + * IMAGE chunk → mtmd_encode_chunk → prefill_embd + * (section-major M-RoPE positions: y→s1, x→s2) + * AUDIO chunk → error (v1: not supported; never silently skip) + * 4. logits requested on the final chunk only + * + * Returns per-branch {tokensDecoded, positionAdvance} — JS cannot know + * multimodal token counts (mtmd owns tokenization) and the trace/pressure + * layers need them. + */ +class StorePrefillMultimodalWorker : public Napi::AsyncWorker { +public: + struct BranchResult { + int64_t tokensDecoded = 0; + int64_t positionAdvance = 0; + }; + + StorePrefillMultimodalWorker(Napi::Env env, + lloyal::branch::BranchStore& store, + mtmd_context* mtmd, + int32_t nEmbdInp, + std::vector handles, + std::vector> sepStorage, + std::vector prompts, + std::vector>> bitmapBytes) + : AsyncWorker(env), _deferred(env), _store(store), _mtmd(mtmd), + _nEmbdInp(nEmbdInp), _handles(std::move(handles)), + _sepStorage(std::move(sepStorage)), _prompts(std::move(prompts)), + _bitmapBytes(std::move(bitmapBytes)) {} + + void Execute() override { + try { + _results.resize(_handles.size()); + for (size_t i = 0; i < _handles.size(); ++i) { + prefillBranch(i); + } + } catch (const std::exception& e) { SetError(e.what()); } + } + + void OnOK() override { + Napi::Env env = Env(); + Napi::Array out = Napi::Array::New(env, _results.size()); + for (size_t i = 0; i < _results.size(); ++i) { + Napi::Object r = Napi::Object::New(env); + r.Set("tokensDecoded", Napi::Number::New(env, static_cast(_results[i].tokensDecoded))); + r.Set("positionAdvance", Napi::Number::New(env, static_cast(_results[i].positionAdvance))); + out.Set(static_cast(i), r); + } + _deferred.Resolve(out); + } + void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + Napi::Promise GetPromise() { return _deferred.Promise(); } + +private: + void prefillBranch(size_t i) { + const auto handle = _handles[i]; + auto* st = _store.get(handle); + if (!st) { + throw std::runtime_error("_storePrefillMultimodal: invalid handle at index " + std::to_string(i)); + } + const llama_pos startPos = st->position; + int64_t tokensDecoded = 0; + + // 1. Leading sep tokens via the token rail + if (!_sepStorage[i].empty()) { + lloyal::branch::DecodeScatterItem item{ + handle, std::span(_sepStorage[i])}; + _store.decode_scatter(std::span(&item, 1)); + tokensDecoded += static_cast(_sepStorage[i].size()); + } + + // 2. Bytes → bitmaps (audio bytes auto-route and are rejected below; + // video files fail to decode with MTMD_VIDEO off — both fail loud) + std::vector bitmaps; + std::vector bitmapPtrs; + bitmaps.reserve(_bitmapBytes[i].size()); + for (const auto& bytes : _bitmapBytes[i]) { + auto wrap = mtmd_helper_bitmap_init_from_buf( + _mtmd, bytes.data(), bytes.size(), /*placeholder*/ false); + if (wrap.video_ctx) { + mtmd_helper_video_free(wrap.video_ctx); + if (wrap.bitmap) mtmd_bitmap_free(wrap.bitmap); + throw std::runtime_error("_storePrefillMultimodal: video input is not supported"); + } + if (!wrap.bitmap) { + throw std::runtime_error( + "_storePrefillMultimodal: unsupported media bytes at image " + + std::to_string(bitmaps.size()) + " (expected jpg/png/bmp/gif)"); + } + bitmaps.emplace_back(wrap.bitmap); + bitmapPtrs.push_back(wrap.bitmap); + } + + // 3. Tokenize: mtmd owns tokenization (no double-tokenize). Flags are + // the text-path parity contract: add_special=false (no + // mid-conversation BOS), parse_special=true (template specials). + mtmd::input_chunks_ptr chunks(mtmd_input_chunks_init()); + mtmd_input_text txt{_prompts[i].c_str(), /*add_special*/ false, /*parse_special*/ true}; + const int32_t rc = mtmd_tokenize(_mtmd, chunks.get(), &txt, + bitmapPtrs.data(), bitmapPtrs.size()); + if (rc == 1) { + throw std::runtime_error( + "_storePrefillMultimodal: media marker count does not match image count"); + } + if (rc != 0) { + throw std::runtime_error("_storePrefillMultimodal: image preprocessing failed"); + } + + // 4. Walk chunks in order + const bool useMrope = mtmd_decode_use_mrope(_mtmd); + const size_t nChunks = mtmd_input_chunks_size(chunks.get()); + for (size_t c = 0; c < nChunks; ++c) { + const mtmd_input_chunk* chunk = mtmd_input_chunks_get(chunks.get(), c); + const auto type = mtmd_input_chunk_get_type(chunk); + const bool isLast = (c == nChunks - 1); + + if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) { + size_t nText = 0; + const llama_token* toks = mtmd_input_chunk_get_tokens_text(chunk, &nText); + if (nText == 0) continue; + lloyal::branch::DecodeScatterItem item{ + handle, std::span(toks, nText)}; + _store.decode_scatter(std::span(&item, 1)); + tokensDecoded += static_cast(nText); + + } else if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { + if (mtmd_encode_chunk(_mtmd, chunk) != 0) { + throw std::runtime_error("_storePrefillMultimodal: image encode failed"); + } + // Context-owned reused buffer — consumed by the decode below + // before the next encode. + const float* embd = mtmd_get_output_embd(_mtmd); + const int32_t nTokens = static_cast(mtmd_input_chunk_get_n_tokens(chunk)); + const llama_pos nPos = mtmd_input_chunk_get_n_pos(chunk); + const llama_pos pos0 = _store.get(handle)->position; + const int32_t nppe = useMrope ? 4 : 1; + + // Section-major positions. M-RoPE section order is t,y,x,z — + // y in section 1, x in section 2 (mtmd's decoder convention). + std::vector pos(static_cast(nTokens) * nppe); + if (useMrope) { + const mtmd_image_tokens* imgToks = mtmd_input_chunk_get_tokens_image(chunk); + if (!imgToks) { + throw std::runtime_error("_storePrefillMultimodal: image tokens missing"); + } + std::vector rel(static_cast(nTokens)); + mtmd_helper_image_get_decoder_pos(imgToks, pos0, rel.data()); + for (int32_t k = 0; k < nTokens; ++k) { + pos[k] = static_cast(rel[k].t); + pos[k + nTokens] = static_cast(rel[k].y); + pos[k + 2 * nTokens] = static_cast(rel[k].x); + pos[k + 3 * nTokens] = static_cast(rel[k].z); + } + } else { + for (int32_t k = 0; k < nTokens; ++k) { + pos[k] = pos0 + k; + } + } + + const bool nonCausal = mtmd_decode_use_non_causal(_mtmd, chunk); + _store.prefill_embd(handle, embd, nTokens, _nEmbdInp, nPos, + pos.data(), nppe, nonCausal, + /*want_logits*/ isLast); + tokensDecoded += static_cast(nTokens); + + } else { + // AUDIO — the byte sniffer auto-routes audio; reject explicitly + // rather than silently skipping KV content the caller expected. + throw std::runtime_error("_storePrefillMultimodal: audio input is not supported"); + } + } + + _results[i].tokensDecoded = tokensDecoded; + _results[i].positionAdvance = + static_cast(_store.get(handle)->position - startPos); + } + + Napi::Promise::Deferred _deferred; + lloyal::branch::BranchStore& _store; + mtmd_context* _mtmd; + int32_t _nEmbdInp; + std::vector _handles; + std::vector> _sepStorage; + std::vector _prompts; + std::vector>> _bitmapBytes; + std::vector _results; +}; + /** * AsyncWorker for batch logit scoring (process_chunks) * Owns token storage and logit output buffers @@ -841,6 +1043,9 @@ Napi::Object SessionContext::Init(Napi::Env env, Napi::Object exports) { // ===== STORE API (internal, wrapped by lib/BranchStore.js) ===== InstanceMethod("_storeCommit", &SessionContext::_storeCommit), InstanceMethod("_storePrefill", &SessionContext::_storePrefill), + InstanceMethod("_storePrefillMultimodal", &SessionContext::_storePrefillMultimodal), + InstanceMethod("supportsVision", &SessionContext::supportsVision), + InstanceMethod("supportsAudio", &SessionContext::supportsAudio), InstanceMethod("_storeMergeLogits", &SessionContext::_storeMergeLogits), InstanceMethod("_storeRetainOnly", &SessionContext::_storeRetainOnly), InstanceMethod("_storeAvailable", &SessionContext::_storeAvailable), @@ -873,6 +1078,11 @@ SessionContext::SessionContext(const Napi::CallbackInfo& info) SessionContext::~SessionContext() { if (!_disposed) { + // Free mtmd first — it holds a reference to the model + if (_mtmdContext) { + mtmd_free(_mtmdContext); + _mtmdContext = nullptr; + } // Free context (depends on model) if (_context) { llama_free(_context); @@ -898,6 +1108,14 @@ void SessionContext::initializeContext( std::cerr << " Shared refcount: " << _model.use_count() << std::endl; } +void SessionContext::initializeMultimodal(struct mtmd_context* mtmd) { + _mtmdContext = mtmd; + std::cerr << "[SessionContext::initializeMultimodal] mmproj attached" + << " (vision=" << (mtmd_support_vision(mtmd) ? "yes" : "no") + << ", audio=" << (mtmd_support_audio(mtmd) ? "yes" : "no") << ")" + << std::endl; +} + Napi::Value SessionContext::tokenize(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); @@ -1263,6 +1481,12 @@ Napi::Value SessionContext::dispose(const Napi::CallbackInfo& info) { // Drain branch store while context is still alive _branchStore.drain(); + // Free mtmd first — it holds a reference to the model + if (_mtmdContext) { + mtmd_free(_mtmdContext); + _mtmdContext = nullptr; + } + // Free context if (_context) { llama_free(_context); @@ -1736,6 +1960,23 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { typeV = t; } + // Extract mmprojPath (optional — enables multimodal input via mtmd) + std::string mmprojPath; + if (options.Has("mmprojPath") && options.Get("mmprojPath").IsString()) { + mmprojPath = options.Get("mmprojPath").As().Utf8Value(); + } + + // Extract image token budget overrides (optional; -1 = model metadata). + // Per-image/per-frame token budgeting — pass-through to mtmd_context_params. + int32_t imageMinTokens = -1; + if (options.Has("imageMinTokens") && options.Get("imageMinTokens").IsNumber()) { + imageMinTokens = options.Get("imageMinTokens").As().Int32Value(); + } + int32_t imageMaxTokens = -1; + if (options.Has("imageMaxTokens") && options.Get("imageMaxTokens").IsNumber()) { + imageMaxTokens = options.Get("imageMaxTokens").As().Int32Value(); + } + // Ensure llama backend is initialized on main thread (thread-safe, once) BackendManager::ensureInitialized(); @@ -1794,6 +2035,32 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { std::cerr << "[CreateContext] Context created successfully" << std::endl; + // Load the mmproj (multimodal projector) BEFORE the model shared_ptr is + // moved into the instance — mtmd_init_from_file validates the projector's + // output dim against the text model and holds a model reference. Fail + // loud: a configured mmproj that cannot load must never silently degrade + // to text-only. + mtmd_context* mtmdCtx = nullptr; + if (!mmprojPath.empty()) { + std::string fsMmprojPath = liblloyal_node::FileSystem::normalizePath(mmprojPath); + if (!liblloyal_node::FileSystem::exists(fsMmprojPath)) { + llama_free(ctx); + throw Napi::Error::New(env, "mmproj file not found: " + fsMmprojPath); + } + mtmd_context_params mparams = mtmd_context_params_default(); + mparams.print_timings = false; + if (nThreads > 0) mparams.n_threads = nThreads; + if (imageMinTokens > 0) mparams.image_min_tokens = imageMinTokens; + if (imageMaxTokens > 0) mparams.image_max_tokens = imageMaxTokens; + std::cerr << "[CreateContext] Loading mmproj: " << fsMmprojPath << std::endl; + mtmdCtx = mtmd_init_from_file(fsMmprojPath.c_str(), sharedModel.get(), mparams); + if (!mtmdCtx) { + llama_free(ctx); + throw Napi::Error::New(env, + "Failed to load mmproj (unsupported projector or corrupt file): " + fsMmprojPath); + } + } + // Create SessionContext instance Napi::Function ctor = env.GetInstanceData()->Value(); Napi::Object instance = ctor.New({}); @@ -1801,6 +2068,9 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { // Initialize obj->initializeContext(std::move(sharedModel), ctx, nBatch); + if (mtmdCtx) { + obj->initializeMultimodal(mtmdCtx); + } std::cerr << "[CreateContext] SessionContext initialized" << std::endl; return instance; @@ -2452,6 +2722,99 @@ Napi::Value SessionContext::_storePrefill(const Napi::CallbackInfo& info) { return worker->GetPromise(); } +Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (!_mtmdContext) { + throw Napi::Error::New(env, + "_storePrefillMultimodal requires a multimodal context — pass mmprojPath to createContext"); + } + if (info.Length() < 4 || !info[0].IsArray() || !info[1].IsArray() || + !info[2].IsArray() || !info[3].IsArray()) { + throw Napi::Error::New(env, + "_storePrefillMultimodal requires (handles[], sepTokens[][], prompts[], bitmaps[][])"); + } + + Napi::Array jsHandles = info[0].As(); + Napi::Array jsSeps = info[1].As(); + Napi::Array jsPrompts = info[2].As(); + Napi::Array jsBitmaps = info[3].As(); + const uint32_t n = jsHandles.Length(); + + if (jsSeps.Length() != n || jsPrompts.Length() != n || jsBitmaps.Length() != n) { + throw Napi::Error::New(env, + "_storePrefillMultimodal: all argument arrays must have the same length"); + } + if (n == 0) { + auto deferred = Napi::Promise::Deferred::New(env); + deferred.Resolve(Napi::Array::New(env, 0)); + return deferred.Promise(); + } + + // Marshal everything on the JS thread — the worker owns copies (Buffers + // must never be touched off-thread). + std::vector handles(n); + std::vector> sepStorage(n); + std::vector prompts(n); + std::vector>> bitmapBytes(n); + + for (uint32_t i = 0; i < n; i++) { + handles[i] = static_cast( + jsHandles.Get(i).As().Uint32Value()); + + Napi::Array jsSep = jsSeps.Get(i).As(); + sepStorage[i].resize(jsSep.Length()); + for (uint32_t j = 0; j < jsSep.Length(); j++) { + sepStorage[i][j] = static_cast( + jsSep.Get(j).As().Int32Value()); + } + + prompts[i] = jsPrompts.Get(i).As().Utf8Value(); + + Napi::Array jsImgs = jsBitmaps.Get(i).As(); + bitmapBytes[i].resize(jsImgs.Length()); + for (uint32_t j = 0; j < jsImgs.Length(); j++) { + Napi::Value v = jsImgs.Get(j); + if (!v.IsBuffer() && !v.IsTypedArray()) { + throw Napi::Error::New(env, + "_storePrefillMultimodal: bitmaps must be Buffer/Uint8Array"); + } + if (v.IsBuffer()) { + auto buf = v.As>(); + bitmapBytes[i][j].assign(buf.Data(), buf.Data() + buf.Length()); + } else { + auto ta = v.As(); + auto* base = static_cast(ta.ArrayBuffer().Data()) + ta.ByteOffset(); + bitmapBytes[i][j].assign(base, base + ta.ByteLength()); + } + } + } + + const int32_t nEmbdInp = llama_model_n_embd_inp(_model.get()); + + auto* worker = new StorePrefillMultimodalWorker( + env, _branchStore, _mtmdContext, nEmbdInp, + std::move(handles), std::move(sepStorage), std::move(prompts), + std::move(bitmapBytes)); + worker->Queue(); + return worker->GetPromise(); +} + +Napi::Value SessionContext::supportsVision(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + return Napi::Boolean::New(env, + _mtmdContext != nullptr && mtmd_support_vision(_mtmdContext)); +} + +Napi::Value SessionContext::supportsAudio(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + return Napi::Boolean::New(env, + _mtmdContext != nullptr && mtmd_support_audio(_mtmdContext)); +} + Napi::Value SessionContext::_storeMergeLogits(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); diff --git a/src/SessionContext.hpp b/src/SessionContext.hpp index ae6442b..8684d17 100644 --- a/src/SessionContext.hpp +++ b/src/SessionContext.hpp @@ -11,6 +11,11 @@ #include #include +// Forward declaration at GLOBAL scope — mtmd_context is llama.cpp/tools/mtmd's +// opaque C type. Declaring it here (not inside the namespace below) keeps the +// member/param types referring to ::mtmd_context, matching . +struct mtmd_context; + namespace liblloyal_node { /** @@ -75,6 +80,14 @@ class SessionContext : public Napi::ObjectWrap { int32_t nBatch = lloyal::defaults::N_BATCH_INIT ); + /** + * Attach the mtmd (multimodal) context loaded from an mmproj file. + * Called by CreateContext when options.mmprojPath is set. Ownership + * transfers to this SessionContext (freed before the model in both + * the destructor and dispose()). + */ + void initializeMultimodal(struct mtmd_context* mtmd); + private: // ===== CORE PRIMITIVES ===== @@ -278,6 +291,21 @@ class SessionContext : public Napi::ObjectWrap { Napi::Value _storeCommit(const Napi::CallbackInfo& info); Napi::Value _storePrefill(const Napi::CallbackInfo& info); + + /** + * Multimodal prefill: per-branch sep tokens + templated prompt (with + * media markers) + image bytes. mtmd tokenizes the prompt, the worker + * walks TEXT/IMAGE chunks in order (token rail / embedding rail). + * Args: (handles: number[], sepTokens: number[][], prompts: string[], + * bitmaps: Buffer[][]) + * Returns: Promise<{tokensDecoded, positionAdvance}[]> + */ + Napi::Value _storePrefillMultimodal(const Napi::CallbackInfo& info); + + /** True when the loaded mmproj has a vision encoder (no mmproj → false). */ + Napi::Value supportsVision(const Napi::CallbackInfo& info); + /** True when the loaded mmproj has an audio encoder (no mmproj → false). */ + Napi::Value supportsAudio(const Napi::CallbackInfo& info); Napi::Value _storeMergeLogits(const Napi::CallbackInfo& info); Napi::Value _storeRetainOnly(const Napi::CallbackInfo& info); Napi::Value _storeAvailable(const Napi::CallbackInfo& info); @@ -292,6 +320,10 @@ class SessionContext : public Napi::ObjectWrap { std::shared_ptr _model; llama_context* _context = nullptr; + /// mtmd multimodal context (owned; nullptr when no mmproj was loaded). + /// Freed BEFORE _model in ~SessionContext and dispose() — mtmd holds a + /// reference to the model. + struct mtmd_context* _mtmdContext = nullptr; bool _disposed = false; int32_t _nBatch = lloyal::defaults::N_BATCH_INIT; diff --git a/test/fixtures/red-square-blue-circle.png b/test/fixtures/red-square-blue-circle.png new file mode 100644 index 0000000000000000000000000000000000000000..ceb92977bbbba27d29cec2f7b60ec3ca5f21e766 GIT binary patch literal 2047 zcmcgtO=uHA6y9C4G_C|vnh3^A2r8RG6>2LK3~o%)He%YTO-1TKX;spQo?0kDBt$9x zfTAE`HD&vkZbU_GduR_*J!nyc{-hPE2L&l5Q1DU^1oh4AY&IJ#5uqOD%bRcBy!X8~ z*_o@Tq0S;W1cqTO!FvA@hGD1hFmu36*6rEBFeR^pes8#YX*Kh#^s;2#YwEOIjc9H6bpkrgOd=Z{o|Hlnc%%S#e# zvJrrZ6=|T;U=^jo!DLGv`|-uh37_75|5%fbC)%w&bM8S-$g?`yx5M^6M?N@{iw~Muek}vvTpo-q_XuLP z!{{pSg59Zp?pmxDR+SL~r}u@Wrp9^olZ6;#SG1i@nZud3<G83Hiuhm$zl+T@cl* zYzDnVbgb16Yw&T~5o)yHb1Vbn?0F^bf`l{~_=|WbBOR1i3*(}kfCrJP295VZ;MSlP z6Yc=G4m=eDpc|G15!|7Z5E>7)9?Gj!0j*`Gzgl)JF^NVJyW}&PmUgHxqNGRE3Wz}J zPb<&)8tDYqjz`bZ=NMJ07ey9&HD0TL$n8({(dw|wh;k>>40l}_G#)NJ-d2@yW!-Ae zNlyhwv~1b72oXp&10rGw8aa)#iBEr&jHVzK5l% z3}jYsU|+=mi0Z+D%?k8Aq6}XMQzD`OOJQM=YUM30AVh`nW!*uysy{&R+F<~qO@+r3 zpn%AzEKlZJB>uG0DToC0sdI47EfbK3CaEQL1wHrTiQCzf3fzg6^rbE0O1e!Dzm1LF z(4Iy8=`)Mk`)PO)4Kyd{Z)=`+KCM2Yzq;64%GbT7yQhxA!_EW)4gMzwPbPi?%!?*Z literal 0 HcmV?d00001 diff --git a/test/integration.ts b/test/integration.ts index 82d80bb..3bed47c 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -2237,6 +2237,252 @@ async function testRerankConcurrent(): Promise { // MAIN // ═══════════════════════════════════════════════════════════════════════════ +// ═══════════════════════════════════════════════════════════════════════════ +// MULTIMODAL (mtmd) TESTS — the embedding rail +// ═══════════════════════════════════════════════════════════════════════════ +// +// Gated on a VL model + mmproj pair being present. Tier selection: +// env LLAMA_VL_MODEL + LLAMA_VL_MMPROJ (explicit), else +// Qwen3.5-4B + its mmproj (local dev — M-RoPE, content assertions), else +// SmolVLM-256M + its mmproj (CI tier — plain positions, mechanics only). + +const VL_PAIRS: Array<{ model: string; mmproj: string; mrope: boolean; strict: boolean }> = [ + { + model: path.join(__dirname, '../models/Qwen3.5-4B-Q4_K_M.gguf'), + mmproj: path.join(__dirname, '../models/mmproj-Qwen3.5-4B-F16.gguf'), + mrope: true, + strict: true, + }, + { + model: path.join(__dirname, '../models/SmolVLM-256M-Instruct-Q8_0.gguf'), + mmproj: path.join(__dirname, '../models/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf'), + mrope: false, + strict: false, + }, +]; + +function pickVlPair(): { model: string; mmproj: string; mrope: boolean; strict: boolean } | null { + if (process.env.LLAMA_VL_MODEL && process.env.LLAMA_VL_MMPROJ) { + const model = path.resolve(process.env.LLAMA_VL_MODEL); + return { + model, + mmproj: path.resolve(process.env.LLAMA_VL_MMPROJ), + mrope: /qwen/i.test(model), + strict: /qwen3\.5-4b|qwen3\.8/i.test(model), + }; + } + for (const p of VL_PAIRS) { + if (fs.existsSync(p.model) && fs.existsSync(p.mmproj)) return p; + } + return null; +} + +const MEDIA_MARKER = '<__media__>'; + +async function testMultimodal(): Promise { + const vl = pickVlPair(); + if (!vl) { + console.log('\n--- Multimodal: SKIPPED (no VL model + mmproj pair in models/) ---'); + return; + } + console.log(`\n--- Multimodal (${path.basename(vl.model)}, mrope=${vl.mrope}) ---`); + + const IMG: Buffer = fs.readFileSync(path.join(__dirname, 'fixtures/red-square-blue-circle.png')); + + // q4_0 KV (assertion 1) + small nBatch so any real image sub-chunks + // (assertion 3 — decode::embd's internal loop is the MAIN path). + const ctx: SessionContext = await addon.createContext({ + modelPath: vl.model, + mmprojPath: vl.mmproj, + nCtx: 8192, + nBatch: 256, + nSeqMax: 8, + typeK: 'q4_0', + typeV: 'q4_0', + nThreads: 4, + } as never); + const mm = ctx as unknown as { + _storePrefillMultimodal( + handles: number[], seps: number[][], prompts: string[], bitmaps: Buffer[][], + ): Promise>; + _storePrefill(handles: number[], tokenArrays: number[][]): Promise; + _storeKvPressure(): { cellsUsed: number }; + supportsVision(): boolean; + supportsAudio(): boolean; + }; + + try { + // Probes + assert(mm.supportsVision() === true, 'supportsVision() → true with mmproj loaded'); + assert(mm.supportsAudio() === false, 'supportsAudio() → false (vision-only mmproj)'); + + // Assertion 2a — marker survives the SYSTEM-ONLY template route (the + // spine-header path; on Qwen3.5 this exercises the sentinel-retry). + const { prompt: sysPrompt } = await ctx.formatChat( + JSON.stringify([{ + role: 'system', + content: [ + { type: 'text', text: 'You can see the attached reference image.' }, + { type: 'media_marker', text: MEDIA_MARKER }, + ], + }]), + { addGenerationPrompt: false } as never, + ); + assert(sysPrompt.includes(MEDIA_MARKER), + 'marker verbatim through system-only route (media_marker content part)'); + + // The main user-turn prompt (generation prompt on — we produce after) + const { prompt: userPrompt } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: 'You are a vision assistant. Answer briefly.' }, + { + role: 'user', + content: [ + { type: 'text', text: 'What shapes and colors are in this image?' }, + { type: 'media_marker', text: MEDIA_MARKER }, + ], + }, + ]), { enableThinking: false } as never); + assert(userPrompt.includes(MEDIA_MARKER), 'marker verbatim through user-turn route'); + + // Assertion 1 + 4 — multimodal prefill on q4_0 KV; cells vs position + const root = Branch.create(ctx, 0, { temperature: 0 }); + const cells0: number = mm._storeKvPressure().cellsUsed; + const res = await mm._storePrefillMultimodal([root.handle], [[]], [userPrompt], [[IMG]]); + const { tokensDecoded, positionAdvance } = res[0]; + assert(tokensDecoded > 0, `prefill decoded ${tokensDecoded} tokens`); + assert(root.position === positionAdvance, + `branch position (${root.position}) == positionAdvance (${positionAdvance})`); + const cells1: number = mm._storeKvPressure().cellsUsed; + assert(cells1 - cells0 === tokensDecoded, + `cells grew by tokensDecoded (${cells1 - cells0} == ${tokensDecoded})`); + if (vl.mrope) { + assert(positionAdvance < tokensDecoded, + `M-RoPE decouple: positionAdvance (${positionAdvance}) < tokensDecoded (${tokensDecoded})`); + } else { + assert(positionAdvance === tokensDecoded, + `plain positions: positionAdvance == tokensDecoded (${tokensDecoded})`); + } + // Assertion 3 — with nBatch 256, a real image (>=256 rows on these + // models) exercised decode::embd's sub-chunk loop; reaching here with a + // coherent KV is the observable (a broken view/pos repack would throw or + // corrupt the grounded answer below). + + // Grounded answer on the image + const gen: number[] = []; + for (let i = 0; i < 48; i++) { + const { token, isStop } = await root.produce(); + if (isStop) break; + await root.commit(token); + gen.push(token); + } + assert(gen.length > 0, `generated ${gen.length} tokens after image prefill`); + const answer: string = await ctx.detokenize(gen); + if (vl.strict) { + assert(/red|square|blue|circle/i.test(answer), + `grounded answer names the content: "${answer.trim()}"`); + } else { + ok(`answer (mechanics tier): "${answer.trim()}"`); + } + const cellsAfterGen: number = mm._storeKvPressure().cellsUsed; + + // Assertion 5 — spine-share: fork ×2 AFTER the image; forks add no cells + const childA = await root.fork(); + const childB = await root.fork(); + assert(mm._storeKvPressure().cellsUsed === cellsAfterGen, + 'fork ×2 adds zero cells (image KV shared, not copied)'); + + const sep: number[] = ctx.getTurnSeparator(); + const { prompt: qPrompt } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: '' }, + { role: 'user', content: 'In one word, what color is the square?' }, + ]), { enableThinking: false } as never); + const qToks: number[] = await ctx.tokenize(qPrompt, false); + const childToks: number[] = [...sep, ...qToks]; + + let aCells = 0; + for (const child of [childA, childB]) { + const before: number = mm._storeKvPressure().cellsUsed; + await child.prefill(childToks); + const cgen: number[] = []; + for (let i = 0; i < 8; i++) { + const { token, isStop } = await child.produce(); + if (isStop) break; + await child.commit(token); + cgen.push(token); + } + assert(cgen.length > 0, 'forked child answers over the shared image'); + if (vl.strict && child === childA) { + const ctext: string = await ctx.detokenize(cgen); + assert(/red/i.test(ctext), `child sees the image (square color): "${ctext.trim()}"`); + } + if (child === childA) aCells = mm._storeKvPressure().cellsUsed - before; + } + + // Assertion 4b — releasing a fork recovers exactly its OWN cells (text + // suffix only, never the shared image): the release-slack fix. + const beforeRelease: number = mm._storeKvPressure().cellsUsed; + await childA.prune(); + const released: number = beforeRelease - mm._storeKvPressure().cellsUsed; + assert(released === aCells, + `release recovered exactly the fork's own cells (${released} == ${aCells})`); + await childB.prune(); + + // Assertion 6 — prefill ending AT the marker (no trailing template + // text): logits must be available for an immediate produce(). + const term = Branch.create(ctx, 0, { temperature: 0 }); + const termRes = await mm._storePrefillMultimodal( + [term.handle], [[]], [`Describe: ${MEDIA_MARKER}`], [[IMG]]); + assert(termRes[0].tokensDecoded > 0, 'marker-terminal prefill decoded'); + const t1 = await term.produce(); + assert(Number.isInteger(t1.token), 'produce() works immediately after marker-terminal prefill'); + await term.prune(); + + // Assertion 7 — duplicate handle in one store call throws (fail loud) + let dupThrew = false; + try { + await mm._storePrefill([root.handle, root.handle], [[qToks[0]], [qToks[0]]]); + } catch { dupThrew = true; } + assert(dupThrew, 'duplicate handle in one _storePrefill rejects'); + + // Marker/bitmap count mismatch throws (mtmd_tokenize rc=1 path) + let mismatchThrew = false; + try { + const b = Branch.create(ctx, 0, { temperature: 0 }); + try { + await mm._storePrefillMultimodal([b.handle], [[]], [`x ${MEDIA_MARKER}`], [[IMG, IMG]]); + } finally { await b.prune(); } + } catch { mismatchThrew = true; } + assert(mismatchThrew, 'marker/bitmap count mismatch rejects'); + + // Assertion 8 — frames (the video-carrying contract): several marker'd + // frames + timestamp text in ONE prefill. + const frames = Branch.create(ctx, 0, { temperature: 0 }); + const framesPrompt = + `Video frames. t=0s: ${MEDIA_MARKER} t=1s: ${MEDIA_MARKER} t=2s: ${MEDIA_MARKER} ` + + `What color is the square across the frames? Answer in one word:`; + const fRes = await mm._storePrefillMultimodal( + [frames.handle], [[]], [framesPrompt], [[IMG, IMG, IMG]]); + assert(fRes[0].tokensDecoded > 0 && fRes[0].positionAdvance > 0, + `frames prefill decoded ${fRes[0].tokensDecoded} tokens (${fRes[0].positionAdvance} positions)`); + const fgen: number[] = []; + for (let i = 0; i < 8; i++) { + const { token, isStop } = await frames.produce(); + if (isStop) break; + await frames.commit(token); + fgen.push(token); + } + assert(fgen.length > 0, 'answer over multi-frame prefill'); + if (vl.strict) { + const ftext: string = await ctx.detokenize(fgen); + assert(/red/i.test(ftext), `temporal answer grounded: "${ftext.trim()}"`); + } + await frames.prune(); + await root.prune(); + } finally { + ctx.dispose(); + } +} + async function main(): Promise { let mainCtx: SessionContext | null = null; @@ -2278,6 +2524,7 @@ async function main(): Promise { await testSetSamplerParams(); await testSetGrammar(); await testBranchMetrics(); + await testMultimodal(); await testRerank(); await testRerankLargeCorpus(); await testRerankConcurrent(); diff --git a/test/matrix.json b/test/matrix.json index ebb68be..db0ff01 100644 --- a/test/matrix.json +++ b/test/matrix.json @@ -54,6 +54,18 @@ "file": "slim-ner-tool.gguf", "url": "https://huggingface.co/llmware/slim-ner-tool/resolve/main/slim-ner-tool.gguf", "usedBy": ["entities"] + }, + { + "name": "SmolVLM-256M", + "file": "SmolVLM-256M-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM-256M-Instruct-GGUF/resolve/main/SmolVLM-256M-Instruct-Q8_0.gguf", + "usedBy": ["multimodal"] + }, + { + "name": "SmolVLM-256M-mmproj", + "file": "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "url": "https://huggingface.co/ggml-org/SmolVLM-256M-Instruct-GGUF/resolve/main/mmproj-SmolVLM-256M-Instruct-Q8_0.gguf", + "usedBy": ["multimodal"] } ] } From 6e4d01ad22512282bd0ce85c5fc1ea1c1ad8840e Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 16:22:26 +1000 Subject: [PATCH 02/54] refactor(mtmd): drive multimodal prefill through the kernel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker walked mtmd chunks itself: encode, position packing, per-segment dispatch and cell bookkeeping, with BranchState::position crossing the binding boundary. Any other SessionContext implementation (Nitro/JSI, a platform encoder) would have had to reimplement the same walk. That logic now lives in liblloyal as BranchStore::decode_segments + MtmdSource, so the worker constructs a source and makes one kernel call: lloyal::MtmdSource source(_mtmd, _prompts[i], _bitmapBytes[i], std::span(_sepStorage[i]), _nEmbdInp); const auto r = _store.decode_segments(_handles[i], source); Net -142/+15 in the worker, and no CMake change was needed — the tell that the boundary is in the right place, since the addon already linked mtmd. Tests: adds the batched fan-out case to the multimodal suite. The existing spine-share block proves forks are free but drives its two children one at a time with the same question; this drives four children together, each asking a different question, through the PUBLIC BranchStore surface — 16 batched store.commit() dispatches, and the JS -> N-API marshalling of parallel handle/token arrays that the kernel-level test cannot reach. fork x4 over the image adds zero cells costs only the text suffixes (96), never the image again 16 batched dispatches, each carrying up to 4 branches "Describe the red object" -> "The red object is a solid square located in the upper left corner of the image" "Describe the blue object" -> "The blue object is a solid circle located in the bottom right corner of the image" The spatial answers only hold if the M-RoPE grid survives the fork. Requires liblloyal#36. --- liblloyal | 2 +- src/SessionContext.cpp | 157 ++++------------------------------------- test/integration.ts | 78 ++++++++++++++++++++ 3 files changed, 94 insertions(+), 143 deletions(-) diff --git a/liblloyal b/liblloyal index 42e5951..dd2e1af 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 42e5951ef26625efbbb51cb95a365838fd78d2ae +Subproject commit dd2e1af6029a8dce64527520808397b87a7783f1 diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index d231ae7..a94fee1 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -677,25 +677,17 @@ class StorePrefillWorker : public Napi::AsyncWorker { }; /** - * AsyncWorker for multimodal prefill (the embedding rail beside the token - * rail). Owns everything it touches off-thread: per-branch sep tokens, - * templated prompt strings (containing <__media__> markers), and copied - * image bytes. All mtmd calls live HERE — liblloyal stays mtmd-free. + * AsyncWorker for multimodal prefill — a marshaller, like its siblings. * - * Per branch, in order (segments are sequential store calls — start_pos is - * read from branch position at dispatch time): - * 1. sep tokens → decode_scatter (token rail) - * 2. mtmd_tokenize → interleaved TEXT/IMAGE chunks - * (add_special=false, parse_special=true — parity with the text path) - * 3. TEXT chunk → decode_scatter (ready token ids; never - * re-tokenized) - * IMAGE chunk → mtmd_encode_chunk → prefill_embd - * (section-major M-RoPE positions: y→s1, x→s2) - * AUDIO chunk → error (v1: not supported; never silently skip) - * 4. logits requested on the final chunk only + * Owns everything it touches off-thread (prompt strings, copied image bytes, + * sep tokens) and hands each branch to the kernel as one call. The pipeline + * itself lives where it belongs: `lloyal::MtmdSource` (mtmd.hpp) produces the + * segment stream, `BranchStore::decode_segments` places it — rails, positions, + * logits and cell/slack accounting are all the store's, so no KV state + * crosses into this layer. * - * Returns per-branch {tokensDecoded, positionAdvance} — JS cannot know - * multimodal token counts (mtmd owns tokenization) and the trace/pressure + * Returns per-branch {tokensDecoded, positionAdvance}: JS cannot know + * multimodal token counts (mtmd owns tokenization) and the trace + pressure * layers need them. */ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { @@ -722,7 +714,11 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { try { _results.resize(_handles.size()); for (size_t i = 0; i < _handles.size(); ++i) { - prefillBranch(i); + lloyal::MtmdSource source( + _mtmd, _prompts[i], _bitmapBytes[i], + std::span(_sepStorage[i]), _nEmbdInp); + const auto r = _store.decode_segments(_handles[i], source); + _results[i] = { r.cells, static_cast(r.advance) }; } } catch (const std::exception& e) { SetError(e.what()); } } @@ -742,129 +738,6 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { Napi::Promise GetPromise() { return _deferred.Promise(); } private: - void prefillBranch(size_t i) { - const auto handle = _handles[i]; - auto* st = _store.get(handle); - if (!st) { - throw std::runtime_error("_storePrefillMultimodal: invalid handle at index " + std::to_string(i)); - } - const llama_pos startPos = st->position; - int64_t tokensDecoded = 0; - - // 1. Leading sep tokens via the token rail - if (!_sepStorage[i].empty()) { - lloyal::branch::DecodeScatterItem item{ - handle, std::span(_sepStorage[i])}; - _store.decode_scatter(std::span(&item, 1)); - tokensDecoded += static_cast(_sepStorage[i].size()); - } - - // 2. Bytes → bitmaps (audio bytes auto-route and are rejected below; - // video files fail to decode with MTMD_VIDEO off — both fail loud) - std::vector bitmaps; - std::vector bitmapPtrs; - bitmaps.reserve(_bitmapBytes[i].size()); - for (const auto& bytes : _bitmapBytes[i]) { - auto wrap = mtmd_helper_bitmap_init_from_buf( - _mtmd, bytes.data(), bytes.size(), /*placeholder*/ false); - if (wrap.video_ctx) { - mtmd_helper_video_free(wrap.video_ctx); - if (wrap.bitmap) mtmd_bitmap_free(wrap.bitmap); - throw std::runtime_error("_storePrefillMultimodal: video input is not supported"); - } - if (!wrap.bitmap) { - throw std::runtime_error( - "_storePrefillMultimodal: unsupported media bytes at image " + - std::to_string(bitmaps.size()) + " (expected jpg/png/bmp/gif)"); - } - bitmaps.emplace_back(wrap.bitmap); - bitmapPtrs.push_back(wrap.bitmap); - } - - // 3. Tokenize: mtmd owns tokenization (no double-tokenize). Flags are - // the text-path parity contract: add_special=false (no - // mid-conversation BOS), parse_special=true (template specials). - mtmd::input_chunks_ptr chunks(mtmd_input_chunks_init()); - mtmd_input_text txt{_prompts[i].c_str(), /*add_special*/ false, /*parse_special*/ true}; - const int32_t rc = mtmd_tokenize(_mtmd, chunks.get(), &txt, - bitmapPtrs.data(), bitmapPtrs.size()); - if (rc == 1) { - throw std::runtime_error( - "_storePrefillMultimodal: media marker count does not match image count"); - } - if (rc != 0) { - throw std::runtime_error("_storePrefillMultimodal: image preprocessing failed"); - } - - // 4. Walk chunks in order - const bool useMrope = mtmd_decode_use_mrope(_mtmd); - const size_t nChunks = mtmd_input_chunks_size(chunks.get()); - for (size_t c = 0; c < nChunks; ++c) { - const mtmd_input_chunk* chunk = mtmd_input_chunks_get(chunks.get(), c); - const auto type = mtmd_input_chunk_get_type(chunk); - const bool isLast = (c == nChunks - 1); - - if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) { - size_t nText = 0; - const llama_token* toks = mtmd_input_chunk_get_tokens_text(chunk, &nText); - if (nText == 0) continue; - lloyal::branch::DecodeScatterItem item{ - handle, std::span(toks, nText)}; - _store.decode_scatter(std::span(&item, 1)); - tokensDecoded += static_cast(nText); - - } else if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { - if (mtmd_encode_chunk(_mtmd, chunk) != 0) { - throw std::runtime_error("_storePrefillMultimodal: image encode failed"); - } - // Context-owned reused buffer — consumed by the decode below - // before the next encode. - const float* embd = mtmd_get_output_embd(_mtmd); - const int32_t nTokens = static_cast(mtmd_input_chunk_get_n_tokens(chunk)); - const llama_pos nPos = mtmd_input_chunk_get_n_pos(chunk); - const llama_pos pos0 = _store.get(handle)->position; - const int32_t nppe = useMrope ? 4 : 1; - - // Section-major positions. M-RoPE section order is t,y,x,z — - // y in section 1, x in section 2 (mtmd's decoder convention). - std::vector pos(static_cast(nTokens) * nppe); - if (useMrope) { - const mtmd_image_tokens* imgToks = mtmd_input_chunk_get_tokens_image(chunk); - if (!imgToks) { - throw std::runtime_error("_storePrefillMultimodal: image tokens missing"); - } - std::vector rel(static_cast(nTokens)); - mtmd_helper_image_get_decoder_pos(imgToks, pos0, rel.data()); - for (int32_t k = 0; k < nTokens; ++k) { - pos[k] = static_cast(rel[k].t); - pos[k + nTokens] = static_cast(rel[k].y); - pos[k + 2 * nTokens] = static_cast(rel[k].x); - pos[k + 3 * nTokens] = static_cast(rel[k].z); - } - } else { - for (int32_t k = 0; k < nTokens; ++k) { - pos[k] = pos0 + k; - } - } - - const bool nonCausal = mtmd_decode_use_non_causal(_mtmd, chunk); - _store.prefill_embd(handle, embd, nTokens, _nEmbdInp, nPos, - pos.data(), nppe, nonCausal, - /*want_logits*/ isLast); - tokensDecoded += static_cast(nTokens); - - } else { - // AUDIO — the byte sniffer auto-routes audio; reject explicitly - // rather than silently skipping KV content the caller expected. - throw std::runtime_error("_storePrefillMultimodal: audio input is not supported"); - } - } - - _results[i].tokensDecoded = tokensDecoded; - _results[i].positionAdvance = - static_cast(_store.get(handle)->position - startPos); - } - Napi::Promise::Deferred _deferred; lloyal::branch::BranchStore& _store; mtmd_context* _mtmd; diff --git a/test/integration.ts b/test/integration.ts index 3bed47c..084d95a 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -2477,6 +2477,84 @@ async function testMultimodal(): Promise { assert(/red/i.test(ftext), `temporal answer grounded: "${ftext.trim()}"`); } await frames.prune(); + + // Assertion 9 — batched fan-out over the shared image, through the PUBLIC + // BranchStore surface. The spine-share above proves forks are free and + // drives its children ONE AT A TIME; this drives N children TOGETHER, each + // asking a different question, at one dispatch per tick. That is the + // README's "N branches, 1 GPU call" over a prefix encoded exactly once — + // and it exercises the JS→N-API marshalling of handle/token arrays that + // the kernel-level test cannot reach. + const store = new BranchStore(ctx); + // Short SENTENCES, not single words: a one-word answer emits its token and + // stops, so the loop below would do a single dispatch and prove nothing + // about sustained batching. Several tokens each keeps all four branches + // live across multiple ticks, which is the property under test. + const asks: Array<{ q: string; expect: RegExp }> = [ + { q: 'Describe the red object in a short sentence.', expect: /square/i }, + { q: 'Describe the blue object in a short sentence.', expect: /circle/i }, + { q: 'In a short sentence, what color is the square?', expect: /red/i }, + { q: 'In a short sentence, what color is the circle?', expect: /blue/i }, + ]; + + const cellsBeforeFan: number = mm._storeKvPressure().cellsUsed; + const kids: InstanceType[] = []; + for (let i = 0; i < asks.length; i++) kids.push(await root.fork()); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan, + `fan-out: fork ×${asks.length} over the image adds zero cells`); + + // Every child's question in ONE variable-length scatter. + const suffixes: number[][] = []; + for (const a of asks) { + const { prompt: p } = await ctx.formatChat(JSON.stringify([ + { role: 'system', content: '' }, + { role: 'user', content: a.q }, + ]), { enableThinking: false } as never); + suffixes.push([...sep, ...(await ctx.tokenize(p, false))]); + } + const suffixTotal: number = suffixes.reduce((n, s) => n + s.length, 0); + await store.prefill(kids.map((b, i) => [b, suffixes[i]] as [typeof b, number[]])); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan + suffixTotal, + `fan-out: costs only the text suffixes (${suffixTotal}), never the image again`); + + // One store.commit() per tick carries every live child. + const fanToks: number[][] = asks.map(() => []); + let ticks = 0; + for (let step = 0; step < 16; step++) { + const entries: Array<[InstanceType, number]> = []; + for (let i = 0; i < kids.length; i++) { + if (fanToks[i].length && fanToks[i][fanToks[i].length - 1] === -1) continue; + const { token, isStop } = await kids[i].produce(); + if (isStop) { fanToks[i].push(-1); continue; } + fanToks[i].push(token); + entries.push([kids[i], token]); + } + if (!entries.length) break; + await store.commit(entries); + ticks++; + } + // >1 is the point: several consecutive ticks each carrying every live + // branch in ONE llama_decode is continuous tree batching. A single + // dispatch would satisfy "it batched" without showing it sustains. + assert(ticks > 1, + `fan-out: ${ticks} batched dispatches, each carrying up to ${kids.length} branches`); + + for (let i = 0; i < asks.length; i++) { + const toks: number[] = fanToks[i].filter((t) => t !== -1); + assert(toks.length > 0, `fan-out: child ${i} produced an answer`); + const text: string = await ctx.detokenize(toks); + if (vl.strict) { + assert(asks[i].expect.test(text), + `fan-out: "${asks[i].q}" → "${text.trim()}"`); + } else { + ok(`fan-out child ${i} (mechanics tier): "${text.trim()}"`); + } + } + + for (const k of kids) await k.prune(); + assert(mm._storeKvPressure().cellsUsed === cellsBeforeFan, + 'fan-out: children release their own cells, image prefix intact'); + await root.prune(); } finally { ctx.dispose(); From 6363fdccfddba8802605bd9be2ccb217024e5e8f Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 16:44:04 +1000 Subject: [PATCH 03/54] fix(ci): account for the two mmproj boot logs in the stderr allowlist e1fb5dc added two boot-scoped stderr writes without bumping the count, so the allowlist gate failed on ubuntu-latest: initializeMultimodal: "mmproj attached (vision=..., audio=...)" CreateContext: "Loading mmproj: " Both fire once at boot, which is what the allowlist exists to permit. 13 -> 15. --- scripts/check-stderr-allowlist.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-stderr-allowlist.js b/scripts/check-stderr-allowlist.js index e243391..1bd6972 100644 --- a/scripts/check-stderr-allowlist.js +++ b/scripts/check-stderr-allowlist.js @@ -25,7 +25,7 @@ const SRC = path.join(__dirname, '..', 'src'); /** file (relative to src/) → exact number of writer call sites. */ const ALLOWLIST = { 'BackendManager.cpp': 2, // once-guarded backend-init provenance + fatal dladdr failure - 'SessionContext.cpp': 13, // initializeContext (4) + CreateContext (9), all boot-scoped + 'SessionContext.cpp': 15, // initializeContext (4) + initializeMultimodal (1) + CreateContext (10), all boot-scoped }; // std::cerr / std::cout streams, fprintf(stderr|stdout, and bare printf — From 6fde0aed7c7eca7c5689bb2d550c213bf9e95e89 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 17:28:34 +1000 Subject: [PATCH 04/54] =?UTF-8?q?fix(mtmd):=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20leak=20guard,=20Uint8Array,=20duplicate=20handles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Copilot review on #49, and picks up liblloyal d84f249. - CreateContext held ctx and mtmdCtx raw until initializeContext / initializeMultimodal took ownership. Anything throwing in between — a missing mmproj, a failed projector load, ctor.New({}), Unwrap — leaked them; the two error paths hand-rolled llama_free(ctx) and the rest did not. An RAII guard now makes cleanup unconditional and both manual frees are gone. The leak predates multimodal (8df3392); mtmd widened it. - _storePrefillMultimodal accepted ANY typed array while the error message said Buffer/Uint8Array, so an Int32Array passed the guard and was reinterpreted as raw bytes. Now checks napi_uint8_array explicitly. - _storePrefillMultimodal did not reject duplicate handles. decode_scatter guards against them, but this path dispatches one handle at a time so the pair never meets there — the same branch would be prefilled twice, each advancing its position. Now fails loud like _storePrefill/_storeCommit. - README: the multimodal snippet used `handle` without creating it, and the documented options are typed through @lloyal-labs/sdk's ContextOptions, so they only typecheck once an SDK carrying them is installed. Says so. Suite: multimodal 22/22 including the fan-out. The one failure is testRerankLargeCorpus, pre-existing and unrelated — fixed in #50. --- README.md | 8 ++++++++ liblloyal | 2 +- src/SessionContext.cpp | 46 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 974699e..b956079 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ const { prompt } = await ctx.formatChat(JSON.stringify([ ]}, ])); +const handle = ctx._branchCreate(0, { temperature: 0 }); const bytes = fs.readFileSync("./photo.jpg"); // jpg/png/bmp/gif const [{ tokensDecoded, positionAdvance }] = await ctx._storePrefillMultimodal([handle], [[]], [prompt], [[bytes]]); @@ -151,6 +152,13 @@ child attends the image with zero re-encode. Several markers with several images in one prefill also works — frames of a video, each preceded by a timestamp, are just that. +> **Types.** `createContext` is typed as `ContextOptions` from +> `@lloyal-labs/sdk`, so `mmprojPath` / `imageMinTokens` / `imageMaxTokens` +> and the `supportsVision()` / `_storePrefillMultimodal()` members only +> typecheck once an SDK carrying the multimodal `ContextOptions` is installed. +> The runtime accepts them regardless; TypeScript consumers on an older SDK +> will need that upgrade first. + Options: `imageMinTokens` / `imageMaxTokens` cap per-image token budgets (default: model metadata). A configured `mmprojPath` that fails to load throws at `createContext` — never a silent fall back to text-only. Audio diff --git a/liblloyal b/liblloyal index dd2e1af..d84f249 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit dd2e1af6029a8dce64527520808397b87a7783f1 +Subproject commit d84f249768550049da1fb662838277d9756b5f8c diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index a94fee1..d6dc0ef 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -1908,6 +1908,20 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { std::cerr << "[CreateContext] Context created successfully" << std::endl; + // ctx and mtmdCtx stay RAW until initializeContext/initializeMultimodal take + // ownership. Everything between here and there can throw — a missing mmproj, + // a failed projector load, ctor.New({}), Unwrap — and each would leak. The + // guard makes cleanup unconditional; release by nulling once ownership moves. + struct NativeHandles { + llama_context* ctx = nullptr; + mtmd_context* mtmd = nullptr; + ~NativeHandles() { + if (mtmd) mtmd_free(mtmd); + if (ctx) llama_free(ctx); + } + } owned; + owned.ctx = ctx; + // Load the mmproj (multimodal projector) BEFORE the model shared_ptr is // moved into the instance — mtmd_init_from_file validates the projector's // output dim against the text model and holds a model reference. Fail @@ -1917,7 +1931,6 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { if (!mmprojPath.empty()) { std::string fsMmprojPath = liblloyal_node::FileSystem::normalizePath(mmprojPath); if (!liblloyal_node::FileSystem::exists(fsMmprojPath)) { - llama_free(ctx); throw Napi::Error::New(env, "mmproj file not found: " + fsMmprojPath); } mtmd_context_params mparams = mtmd_context_params_default(); @@ -1928,10 +1941,10 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { std::cerr << "[CreateContext] Loading mmproj: " << fsMmprojPath << std::endl; mtmdCtx = mtmd_init_from_file(fsMmprojPath.c_str(), sharedModel.get(), mparams); if (!mtmdCtx) { - llama_free(ctx); throw Napi::Error::New(env, "Failed to load mmproj (unsupported projector or corrupt file): " + fsMmprojPath); } + owned.mtmd = mtmdCtx; } // Create SessionContext instance @@ -1941,8 +1954,10 @@ Napi::Value CreateContext(const Napi::CallbackInfo& info) { // Initialize obj->initializeContext(std::move(sharedModel), ctx, nBatch); + owned.ctx = nullptr; // ownership transferred if (mtmdCtx) { obj->initializeMultimodal(mtmdCtx); + owned.mtmd = nullptr; } std::cerr << "[CreateContext] SessionContext initialized" << std::endl; @@ -2625,6 +2640,26 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in return deferred.Promise(); } + // Duplicate handles would prefill the same branch twice in sequence, each + // advancing its position. decode_scatter rejects duplicates, but this path + // dispatches one handle at a time so the pair never meets there — check it + // here to keep the same fail-loud contract as _storePrefill/_storeCommit. + { + std::vector seen; + seen.reserve(n); + for (uint32_t i = 0; i < n; i++) { + const double h = jsHandles.Get(i).As().DoubleValue(); + for (uint32_t j = 0; j < seen.size(); j++) { + if (seen[j] == h) { + throw Napi::Error::New(env, + "_storePrefillMultimodal: duplicate handle at indices " + + std::to_string(j) + " and " + std::to_string(i)); + } + } + seen.push_back(h); + } + } + // Marshal everything on the JS thread — the worker owns copies (Buffers // must never be touched off-thread). std::vector handles(n); @@ -2649,7 +2684,12 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in bitmapBytes[i].resize(jsImgs.Length()); for (uint32_t j = 0; j < jsImgs.Length(); j++) { Napi::Value v = jsImgs.Get(j); - if (!v.IsBuffer() && !v.IsTypedArray()) { + // Uint8Array specifically, not any TypedArray: an Int32Array would + // pass a bare IsTypedArray() check and then be reinterpreted as raw + // bytes, silently feeding the decoder the wrong buffer. + const bool isU8 = v.IsTypedArray() && + v.As().TypedArrayType() == napi_uint8_array; + if (!v.IsBuffer() && !isU8) { throw Napi::Error::New(env, "_storePrefillMultimodal: bitmaps must be Buffer/Uint8Array"); } From 3794f37e16aceab77ad078c1806f980510fe5b07 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 17:57:54 +1000 Subject: [PATCH 05/54] chore(liblloyal): bump to b9be568 (review fixes) Picks up the non-causal single-dispatch requirement, the CausalGuard RAII restore, segment-geometry validation, the stale-logits clear, and the stub-tier slack coverage. --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index d84f249..b9be568 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit d84f249768550049da1fb662838277d9756b5f8c +Subproject commit b9be568834f738f8c2608c089a13c58f85430999 From 8078b99829c564acc6e5e968e5f275eb8a19db42 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 18:24:39 +1000 Subject: [PATCH 06/54] chore(liblloyal): bump to 12e5bb8 (second review round) Embedding-width validation against the resident model, empty-segment rejection so terminality is correct by construction, and audio rejected in MtmdSource's constructor before any dispatch. --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index b9be568..12e5bb8 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit b9be568834f738f8c2608c089a13c58f85430999 +Subproject commit 12e5bb876f5cc89b2f3077899f5198ec54672c81 From 02b4ff8d9c4fb72a60bbb314a1e4fa6937db6125 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 18:51:33 +1000 Subject: [PATCH 07/54] chore(liblloyal): bump to a58970e (README overhaul) Diagram-forward README with the multimodal architecture section, plus three non-compiling code samples fixed (wrong item type, a reseed_chain overload that does not exist, wrong CMake target). --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 12e5bb8..a58970e 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 12e5bb876f5cc89b2f3077899f5198ec54672c81 +Subproject commit a58970e46eda9ac8c03333d02dbf9d33003f70a5 From b0034996a7f38b25888ee93aa8f21f01c1346a23 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:02:23 +1000 Subject: [PATCH 08/54] chore(liblloyal): bump to 1688358 (README corrections) Slot/lease model corrected (allocate takes both atomically; n_seq_max is the real bound), MtmdSource drawn as implementing SegmentSource rather than feeding it, decode_each's one-dispatch claim qualified, and two API instructions fixed (nonexistent v0.1.0 tag, ForkOpts on branch::fork). --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index a58970e..1688358 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit a58970e46eda9ac8c03333d02dbf9d33003f70a5 +Subproject commit 1688358835fcfbbbc1ac45a504c3a4e6138928dd From d75888bf93c2e2599c2fe8ef39798b14f04fe032 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:06:33 +1000 Subject: [PATCH 09/54] chore(liblloyal): bump to 273e808 (decode_embd rollback) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 1688358..273e808 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 1688358835fcfbbbc1ac45a504c3a4e6138928dd +Subproject commit 273e8081b6891a2089b3cd427b75608f3ffcf6ac From d5ff2f4f366abd4a9e71fcc7f959d44ef5503714 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:24:58 +1000 Subject: [PATCH 10/54] chore(liblloyal): bump to d08d444 (contract tests + sanitizer CI) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 273e808..d08d444 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 273e8081b6891a2089b3cd427b75608f3ffcf6ac +Subproject commit d08d44432694b3043593c9f7d3d01137e31872a4 From 3de4e8d01188307b34a9761ad635ef526d4500c3 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:26:17 +1000 Subject: [PATCH 11/54] chore(liblloyal): bump to 7b28a62 (README positioning) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index d08d444..7b28a62 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit d08d44432694b3043593c9f7d3d01137e31872a4 +Subproject commit 7b28a621f39f47142c79eb826340ee4ff6e75e30 From 9140c2dc275c6c0f94244ac3abce7b5f7db6e94d Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:36:43 +1000 Subject: [PATCH 12/54] chore(liblloyal): bump to 1643c97 (merge semantics in README) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 7b28a62..1643c97 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 7b28a621f39f47142c79eb826340ee4ff6e75e30 +Subproject commit 1643c97556a914161c90f5c3b807664a090647ce From a164daa983610b23a872c9351f47da8cbb3f0fcf Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:50:07 +1000 Subject: [PATCH 13/54] chore(liblloyal): bump to abc2963 (Git table corrections) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 1643c97..abc2963 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 1643c97556a914161c90f5c3b807664a090647ce +Subproject commit abc2963383faee18f6b26bed07a9e68136b1b97f From 322b399c0a55832865cd3b11c55512b48cdef820 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:55:43 +1000 Subject: [PATCH 14/54] chore(liblloyal): bump to e980952 (README math block) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index abc2963..e980952 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit abc2963383faee18f6b26bed07a9e68136b1b97f +Subproject commit e9809528ab40970422f191ad463a06fde3afd57e From f24fe7be9be0ddc57b73ecca9ebf231b0a2d4d98 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:56:42 +1000 Subject: [PATCH 15/54] chore(liblloyal): bump to 1f90163 (DExperts citation) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index e980952..1f90163 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit e9809528ab40970422f191ad463a06fde3afd57e +Subproject commit 1f90163520ec52d0dbd4d7a911945f5815824b29 From aadc01b0de4854ed53ba491246a6c0f91c8890db Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 19:59:10 +1000 Subject: [PATCH 16/54] chore(liblloyal): bump to a78ff45 (opening line) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 1f90163..a78ff45 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 1f90163520ec52d0dbd4d7a911945f5815824b29 +Subproject commit a78ff4591dd8be4bca3929bbce68fdf14d39c433 From 43fd13ed17f93cdc7886da9ba59b4a0d99689ded Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:00:12 +1000 Subject: [PATCH 17/54] chore(liblloyal): bump to 8d45226 (README de-duplication) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index a78ff45..8d45226 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit a78ff4591dd8be4bca3929bbce68fdf14d39c433 +Subproject commit 8d452267732df6d943af1b85bbceca74aca3eecb From f533da316e5364425d25d6a33db598c2abf49d93 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:05:20 +1000 Subject: [PATCH 18/54] chore(liblloyal): bump to 6fc967b (recurrent rollback soundness) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 8d45226..6fc967b 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 8d452267732df6d943af1b85bbceca74aca3eecb +Subproject commit 6fc967b12aeec13c2d5e737dbccd5f34aef5f49c From cc36b5280d69a46095f669780e1806304ec8e0f6 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:20:15 +1000 Subject: [PATCH 19/54] chore(liblloyal): bump to f67b209 (prefill cost + rollback removal) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 6fc967b..f67b209 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 6fc967b12aeec13c2d5e737dbccd5f34aef5f49c +Subproject commit f67b20988b8c9a60be95bc8145819cb193e605ff From 2be7647f50d130227a3040ff50ec2af846a2d28f Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:27:51 +1000 Subject: [PATCH 20/54] chore(liblloyal): bump to 8859de9 (KV landmine docs) --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index f67b209..8859de9 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit f67b20988b8c9a60be95bc8145819cb193e605ff +Subproject commit 8859de929de10c06bcad19dff8705caa3d09b0dd From 4b37b4f9fc721ea3108d93d00f080fb001e02b48 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:41:12 +1000 Subject: [PATCH 21/54] =?UTF-8?q?docs(readme):=20revamp,=20carrying=20libl?= =?UTF-8?q?loyal's=20register=20=E2=80=94=20and=20fix=20five=20broken=20sa?= =?UTF-8?q?mples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same treatment as liblloyal: say what the layer is FOR, diagram only the genuinely hard structure, and verify every claim against source instead of restating it. Five things in the old README did not survive checking: - examples/best-of-n/ does not exist. Only chat, embed and entropy do, so the one runnable command in the file did not run. - The CI table listed "Ministral 3B / mistral". test/matrix.json has GLM-Edge. - formatChat takes a JSON STRING; the sample passed an object. The SDK's own docs call the parameter messagesJson throughout. - loadBinary takes the variant directly — loadBinary("cuda") — not loadBinary({ gpuVariant: "cuda" }). - Branch.produce() is async, so the Quick Start's { b, ...b.produce() } spread a Promise and produced neither token nor isStop. That cohort loop wants produceSync(), which is the point of batching: sample everyone, commit once. New material where the old file was thin: - "Which binary loads" — the resolution order is what people actually hit when the wrong binary loads, and it was undocumented. Diagrammed, with the four environment variables and the deliberate no-fallthrough on a corrupt cache. - "Who owns what" — the layer stack, and the seam at lloyal.node that lets nitro-llama serve React Native from the same kernel. - Multimodal now explains why positionAdvance < tokensDecoded rather than only showing the call, and carries the SDK-version caveat. Platform count (13) verified against release.yml's matrix legs. --- README.md | 246 ++++++++++++++++++++++++++---------------------------- 1 file changed, 117 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index b956079..738ae98 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ [![License](https://img.shields.io/badge/license-FSL--1.1--Apache--2.0-blue.svg)](LICENSE) [![llama.cpp](https://img.shields.io/badge/llama.cpp-b9581-green.svg)](https://github.com/ggml-org/llama.cpp/releases/tag/b9581) -**Native backend for the lloyal inference platform.** +**Vertical Inference on Node — the kernel prebuilt for 13 targets, GPU chosen at run time** -Prebuilt llama.cpp binaries for 13 platform/GPU combinations, exposing a `SessionContext` that powers the [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) inference primitives (Branch, BranchStore, Session, Rerank) and [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) multi-agent framework. Built on [liblloyal](https://github.com/lloyal-ai/liblloyal), a header-only C++20 inference kernel for llama.cpp. +[liblloyal](https://github.com/lloyal-ai/liblloyal) is the C++20 kernel: Git-like tree ops over live inference state. This package is how you run it. One `npm install` gets a binary compiled for your platform, a `SessionContext` bound to it, and the rest of the HDK re-exported — so `import { Branch, useAgent } from "@lloyal-labs/lloyal.node"` works without a second package. -All SDK and agent exports are re-exported from this package for convenience — `import { Branch, useAgent, agentPool } from "@lloyal-labs/lloyal.node"` works out of the box. +Nothing compiles on install. The variant that matches your hardware is chosen when the process starts, so the same artifact ships to a CPU laptop and a CUDA box. ## Install @@ -18,8 +18,6 @@ All SDK and agent exports are re-exported from this package for convenience — npm install @lloyal-labs/lloyal.node ``` -Prebuilt binaries for 13 platform/GPU combinations. GPU selection at runtime, not install time. - | Platform | Arch | Acceleration | | -------- | ----- | ------------------- | | macOS | arm64 | Metal | @@ -29,7 +27,34 @@ Prebuilt binaries for 13 platform/GPU combinations. GPU selection at runtime, no | Windows | x64 | CPU / CUDA / Vulkan | | Windows | arm64 | CPU / Vulkan | -## Quick Start +## Which binary loads + +Resolution is ordered and mostly invisible — but when the wrong binary loads, this is the order that decided it. + +```mermaid +flowchart TD + L{"LLOYAL_LOCAL=1"} -->|yes| LB["build/Release
throws if absent — never falls back"] + L -->|no| D{"LLOYAL_BACKEND_DIR"} + D -->|set| DP["that pack — asserts the devices you asked for"] + D -->|unset| C{"cached backend pack"} + C -->|valid| CP["use it"] + C -->|none| V{"variant requested?
argument or LLOYAL_GPU"} + V -->|yes| VP["platform package for that variant"] + VP -->|"fails"| W["warn, fall back
unless LLOYAL_NO_FALLBACK=1"] + V -->|no| DEF["local build, then the default CPU package"] + W --> DEF +``` + +| Variable | Effect | +| --- | --- | +| `LLOYAL_GPU` | Variant to try — same values as the `loadBinary()` argument | +| `LLOYAL_NO_FALLBACK=1` | A failed variant throws instead of warning and dropping to CPU | +| `LLOYAL_LOCAL=1` | Force `build/Release`; fails loudly rather than silently using a published binary | +| `LLOYAL_BACKEND_DIR` | Load a backend pack from a named directory | + +An invalid cached pack **throws** rather than falling through to npm — a corrupt cache is a bug to see, not to route around. + +## Quick start ```javascript import { createContext } from "@lloyal-labs/lloyal.node"; @@ -41,88 +66,92 @@ const store = new BranchStore(ctx); const root = Branch.create(ctx, 0, { temperature: 0.8 }); await root.prefill(await ctx.tokenize("Explain quantum entanglement")); -// Fork and generate — all branches in lockstep, 1 GPU call per step +// Fork three ways; every live branch advances in one GPU call per step const branches = await Promise.all([root.fork(), root.fork(), root.fork()]); for (;;) { const live = branches.filter((b) => !b.disposed); if (!live.length) break; - const produced = live.map((b) => ({ b, ...b.produce() })); + + const produced = live.map((b) => ({ b, ...b.produceSync() })); for (const p of produced.filter((p) => p.isStop)) await p.b.prune(); + const items = produced .filter((p) => !p.isStop) - .map((p) => { - p.b.accept(p.token); - return [p.b, p.token]; - }); - await store.commit(items); + .map((p) => { p.b.accept(p.token); return [p.b, p.token]; }); + if (items.length) await store.commit(items); // N branches, 1 llama_decode() } ``` -Or for single-branch generation, Branch is an async iterable: +`produceSync()` samples without awaiting so the whole cohort can be collected and committed together — that batching is the point. `await branch.produce()` is the single-branch form. + +For one branch, `Branch` is an async iterable: ```javascript -for await (const { token, text } of branch) { - process.stdout.write(text); -} +for await (const { token, text } of branch) process.stdout.write(text); ``` -See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) for the full Branch API, continuous tree batching, KV tenancy, and topology documentation. +See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) for the Branch API, continuous tree batching, KV tenancy and topology. -### Without the SDK +## Who owns what -`createContext` returns a `SessionContext` — the native interface to llama.cpp. You can use it directly without the SDK's Branch/BranchStore layer: +```mermaid +flowchart TD + A["@lloyal-labs/rig
Apps, retrieval, framework tools"] --> B["@lloyal-labs/lloyal-agents
agents, pools, spines"] + B --> C["@lloyal-labs/sdk
Branch · BranchStore · Session · Rerank"] + C --> D["lloyal.node
SessionContext · binaries · GPU selection"] + D --> E["liblloyal
C++20 kernel — the tree ops"] + E --> F["llama.cpp b9581"] +``` -```javascript -import { createContext } from "@lloyal-labs/lloyal.node"; +Everything above `lloyal.node` is backend-agnostic; everything below is native. That seam is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. + +**Native-only, not in the SDK:** + +- `createContext(options)` — load a GGUF, get a `SessionContext`. `mmprojPath` loads a multimodal projector beside it. +- `_storePrefillMultimodal(...)` — image + text into a branch's KV, plus `supportsVision()` / `supportsAudio()` +- `loadBinary(variant?)` — pick the variant explicitly +- The prebuilt binaries themselves + +**Re-exported, so one install is enough:** `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics — and from the agents package `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, plus the App protocol surfaces that pair with rig's `defineApp`. +## The native surface + +`createContext` returns a `SessionContext` — llama.cpp as this package exposes it. The SDK's `Branch`/`BranchStore` wrap these; you can use them directly. + +```javascript const ctx = await createContext({ modelPath: "./model.gguf", nSeqMax: 4 }); -// Chat templates — model-agnostic formatting + tool calling -const { prompt, grammar, format } = await ctx.formatChat(messages, { - addGenerationPrompt: true, - tools: [{ type: "function", function: { name: "search", parameters: schema } }], -}); +// Chat templates — model-agnostic formatting and tool calling. +// NOTE: messages go in as a JSON STRING, not an object. +const { prompt, grammar, format } = await ctx.formatChat( + JSON.stringify([{ role: "user", content: "hello" }]), + { addGenerationPrompt: true, + tools: [{ type: "function", function: { name: "search", parameters: schema } }] }, +); const { content, toolCalls } = await ctx.parseChatOutput(output, format); -// Branch primitives — what the SDK's Branch class wraps +// Branch primitives — what Branch wraps const handle = ctx._branchCreate(0, samplerParams); await ctx._branchPrefill(handle, tokens); const token = ctx._branchSample(handle); -const text = ctx.tokenToText(token); -const isStop = ctx.isStopToken(token); ctx._branchAccept(handle, token); -const logits = ctx._branchGetLogits(handle); // Float32Array(vocabSize) -const entropy = ctx._branchModelEntropy(handle); +const logits = ctx._branchGetLogits(handle); // Float32Array(vocabSize) const child = ctx._branchFork(handle); -// Store primitives — what the SDK's BranchStore wraps -await ctx._storeCommit([handle1, handle2], [tok1, tok2]); // N branches, 1 GPU call +// Store primitives — what BranchStore wraps +await ctx._storeCommit([handle1, handle2], [tok1, tok2]); // N branches, 1 GPU call await ctx._storePrefill([handle], [tokens]); await ctx._storeRetainOnly(winner); -const available = ctx._storeAvailable(); -// KV cache — snapshot, copy, persist -await ctx.kvSeqCopy(0, 1); // share prefix across sequences -await ctx.kvCacheSave(); // snapshot for rollback -await ctx.kvCacheLoad(); // restore checkpoint -await ctx.kvCacheWriteFile("cache.bin"); // persist to disk - -// Embeddings +// KV, embeddings, grammar +await ctx.kvSeqCopy(0, 1); const embeddings = await ctx.encode("query text"); -const dim = ctx.getEmbeddingDimension(); - -// Grammar + tokenizer const grammar = await ctx.jsonSchemaToGrammar(schema); -const tokens = await ctx.tokenize("Hello world"); -const sep = await ctx.getTurnSeparator(); ``` -## Multimodal (Vision) +## Multimodal -Load a model's multimodal projector (`mmproj` GGUF) alongside it and prefill -images into any branch's KV. Works with any llama.cpp-supported VL model — -the projector decides the position mode (M-RoPE for Qwen, plain for -llava-style) at runtime. +An image is decoded, projected into the model's native input embeddings, and admitted through `llama_batch.embd` beside the token stream. After that it is ordinary KV: fork the branch and every child attends the image with no re-encode. ```javascript const ctx = await createContext({ @@ -130,9 +159,9 @@ const ctx = await createContext({ mmprojPath: "./mmproj-F16.gguf", nSeqMax: 8, }); -ctx.supportsVision(); // true +ctx.supportsVision(); // true -// One <__media__> marker per image, injected as a media_marker content part +// One <__media__> marker per image, as a media_marker content part const { prompt } = await ctx.formatChat(JSON.stringify([ { role: "user", content: [ { type: "text", text: "What is in this image?" }, @@ -141,115 +170,74 @@ const { prompt } = await ctx.formatChat(JSON.stringify([ ])); const handle = ctx._branchCreate(0, { temperature: 0 }); -const bytes = fs.readFileSync("./photo.jpg"); // jpg/png/bmp/gif +const bytes = fs.readFileSync("./photo.jpg"); // jpg/png/bmp/gif const [{ tokensDecoded, positionAdvance }] = await ctx._storePrefillMultimodal([handle], [[]], [prompt], [[bytes]]); -// ...then produce/commit as usual — the branch attends the image ``` -The image lands in the KV as a shared prefix: fork the branch and every -child attends the image with zero re-encode. Several markers with several -images in one prefill also works — frames of a video, each preceded by a -timestamp, are just that. - -> **Types.** `createContext` is typed as `ContextOptions` from -> `@lloyal-labs/sdk`, so `mmprojPath` / `imageMinTokens` / `imageMaxTokens` -> and the `supportsVision()` / `_storePrefillMultimodal()` members only -> typecheck once an SDK carrying the multimodal `ContextOptions` is installed. -> The runtime accepts them regardless; TypeScript consumers on an older SDK -> will need that upgrade first. +`positionAdvance < tokensDecoded` under M-RoPE — an image costs more KV cells than it advances position, and the kernel tracks the gap so pressure accounting stays exact. Several markers with several images in one prefill also works; video frames with timestamps are exactly that. -Options: `imageMinTokens` / `imageMaxTokens` cap per-image token budgets -(default: model metadata). A configured `mmprojPath` that fails to load -throws at `createContext` — never a silent fall back to text-only. Audio -bytes are rejected explicitly (no audio surface yet). +> **Types.** `createContext` is typed as `ContextOptions` from `@lloyal-labs/sdk`, so `mmprojPath` / `imageMinTokens` / `imageMaxTokens` and the `supportsVision()` / `_storePrefillMultimodal()` members only typecheck once an SDK carrying multimodal `ContextOptions` is installed. The runtime accepts them regardless. -## What This Package Provides +A configured `mmprojPath` that fails to load throws at `createContext` — never a silent fall back to text-only. Audio is rejected explicitly. -**Native-only** (not in SDK): - -- `createContext(options)` — load a GGUF model, return a `SessionContext`; - `mmprojPath` loads the multimodal projector beside it -- `_storePrefillMultimodal(...)` — image+text prefill into a branch's KV - (the embedding rail); `supportsVision()` / `supportsAudio()` probes -- `loadBinary(options?)` — explicit GPU variant selection with automatic fallback -- Prebuilt binaries for 13 platform/GPU combinations - -**Re-exported from [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk):** - -- `Branch`, `BranchStore`, `Session`, `Rerank` -- Per-token metrics: `modelEntropy()`, `modelSurprisal()`, `samplingPerplexity` -- Chat formatting: `formatChat()`, `parseChatOutput()` -- Grammar: `jsonSchemaToGrammar()`, `setGrammar()` - -**Re-exported from [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents):** - -- `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit` -- Structured concurrency DAG via Effection generators -- In-loop orchestration: agents as branches of a single running process -- App protocol surfaces (`AppRegistryCtx`, `AppConfigStoreCtx`, `App`, `AppManifest`) when paired with [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig)'s `defineApp` / `createAppRegistry` - -## GPU Variant Selection +## GPU variant selection ```javascript import { loadBinary, createContext } from "@lloyal-labs/lloyal.node"; -// Automatic — uses Metal on macOS, CPU elsewhere +// Automatic — Metal on macOS arm64, CPU elsewhere const ctx = await createContext({ modelPath: "./model.gguf" }); -// Explicit CUDA -const binding = loadBinary({ gpuVariant: "cuda" }); -const ctx = await binding.createContext({ modelPath: "./model.gguf" }); -// Falls back to CPU with a warning if CUDA runtime not available +// Explicit: loadBinary takes the variant directly +const binding = loadBinary("cuda"); // "default" | "cuda" | "vulkan" +const ctx2 = await binding.createContext({ modelPath: "./model.gguf" }); +// Falls back to CPU with a warning unless LLOYAL_NO_FALLBACK=1 ``` ## Examples | Example | Pattern | | --------------------------------- | ------------------------------------------------- | -| [`entropy/`](./examples/entropy/) | `modelEntropy()` mid-generation as control signal | | [`chat/`](./examples/chat/) | Interactive streaming chat | -| [`embed/`](./examples/embed/) | Text embeddings extraction | +| [`embed/`](./examples/embed/) | Text embedding extraction | +| [`entropy/`](./examples/entropy/) | `modelEntropy()` mid-generation as a control signal | ```bash -npx tsx examples/best-of-n/best-of-n.ts npx tsx examples/chat/chat.ts ./model.gguf ``` -## CI Testing - -Integration tests run real inference across architectures: +## CI -| Architecture | Test Model | Template | -| ------------ | ------------ | -------- | -| Llama | Llama 3.2 1B | llama3 | -| Phi | Phi 3.5 Mini | phi3 | -| Qwen | Qwen 3 1.7B | chatml | -| Gemma | Gemma 3 1B | gemma | -| SmolLM | SmolLM2 1.7B | chatml | -| Ministral | Ministral 3B | mistral | +Integration tests run real inference across architectures, so a template regression surfaces as a wrong answer rather than a clean pass: -Multimodal tests run on two tiers: SmolVLM-256M (plain positions, CI) and -Qwen3.5-4B + mmproj (M-RoPE, local/GPU rig). +| Model | Template | +| ------------ | ---------- | +| SmolLM2 1.7B | chatml *(default)* | +| Llama 3.2 | llama3 | +| Phi 3.5 | phi3 | +| Qwen3 | chatml | +| Gemma 3 | gemma | +| GLM-Edge | glm-edge | -See [distribution.md](docs/distribution.md) for details. +Multimodal runs two tiers: SmolVLM-256M for plain positions in CI, Qwen3.5-4B + mmproj for M-RoPE locally and on the GPU rig. See [distribution.md](docs/distribution.md). ## Ecosystem -| Package | Description | -| ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | -| [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives (Branch, BranchStore, Session, Rerank) | -| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + App protocol primitives | -| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | App protocol helpers, retrieval providers, framework tools (Plan/Delegate/Report) | -| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses + Apps; publish/install signed Apps via the channel | -| [liblloyal](https://github.com/lloyal-ai/liblloyal) | Header-only C++20 inference kernel for llama.cpp | -| **lloyal.node** | This package — native backend + prebuilt binaries | -| [nitro-llama](https://github.com/lloyal-ai/nitro-llama) | React Native backend via Nitro Modules | -| [tsampler](https://github.com/lloyal-ai/tsampler) | Reference sampler implementation | +| Package | Description | +| --- | --- | +| [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives | +| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + App protocol primitives | +| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | App helpers, retrieval providers, framework tools | +| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses and Apps, publish/install signed Apps | +| [liblloyal](https://github.com/lloyal-ai/liblloyal) | The C++20 kernel | +| **lloyal.node** | This package — native backend + prebuilt binaries | +| [nitro-llama](https://github.com/lloyal-ai/nitro-llama) | React Native backend via Nitro Modules | +| [tsampler](https://github.com/lloyal-ai/tsampler) | Reference sampler implementation | ## Contributing -See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup and release process. +See [CONTRIBUTING.md](./CONTRIBUTING.md) for development setup and the release process. ## License From a10a524db0efca701b182a2dcd91d483f56e4564 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:42:35 +1000 Subject: [PATCH 22/54] docs(readme): the resolution order is a list, not a flowchart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rendered it and looked: five chained decision diamonds sprawled down the page with node text clipped ('never fal', 'the devi'). An ordered fallback chain is a sequence — a numbered table says it in a fraction of the height and carries the failure semantics per step, which the diagram could not. Dropped the environment-variable table and the closing sentence with it: both restated rows the ordered table already covers. --- README.md | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 738ae98..b650c2b 100644 --- a/README.md +++ b/README.md @@ -31,28 +31,15 @@ npm install @lloyal-labs/lloyal.node Resolution is ordered and mostly invisible — but when the wrong binary loads, this is the order that decided it. -```mermaid -flowchart TD - L{"LLOYAL_LOCAL=1"} -->|yes| LB["build/Release
throws if absent — never falls back"] - L -->|no| D{"LLOYAL_BACKEND_DIR"} - D -->|set| DP["that pack — asserts the devices you asked for"] - D -->|unset| C{"cached backend pack"} - C -->|valid| CP["use it"] - C -->|none| V{"variant requested?
argument or LLOYAL_GPU"} - V -->|yes| VP["platform package for that variant"] - VP -->|"fails"| W["warn, fall back
unless LLOYAL_NO_FALLBACK=1"] - V -->|no| DEF["local build, then the default CPU package"] - W --> DEF -``` - -| Variable | Effect | -| --- | --- | -| `LLOYAL_GPU` | Variant to try — same values as the `loadBinary()` argument | -| `LLOYAL_NO_FALLBACK=1` | A failed variant throws instead of warning and dropping to CPU | -| `LLOYAL_LOCAL=1` | Force `build/Release`; fails loudly rather than silently using a published binary | -| `LLOYAL_BACKEND_DIR` | Load a backend pack from a named directory | +| # | Source | If it fails | +| --- | --- | --- | +| 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | +| 2 | `LLOYAL_BACKEND_DIR` → that pack | throws; asserts the devices you asked for | +| 3 | a cached backend pack | throws if present-but-invalid — **no** fallthrough to npm | +| 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | +| 5 | local `build/Release` | continues — fresher than an installed package during development | +| 6 | default CPU package for the platform | throws, naming everything it tried | -An invalid cached pack **throws** rather than falling through to npm — a corrupt cache is a bug to see, not to route around. ## Quick start From 517c11faa149f41f5796a586bfbbadc473247ae6 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:43:28 +1000 Subject: [PATCH 23/54] docs(readme): stop the layer diagram clipping its labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rendered it: mermaid did not widen the boxes to fit, so 'Branch · BranchStore · Session · Rerank' and 'SessionContext · binaries · GPU selection' were cut mid-word. Split across more
lines, none longer than the shortest label that rendered whole. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b650c2b..8bfd30e 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,10 @@ See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk ```mermaid flowchart TD - A["@lloyal-labs/rig
Apps, retrieval, framework tools"] --> B["@lloyal-labs/lloyal-agents
agents, pools, spines"] - B --> C["@lloyal-labs/sdk
Branch · BranchStore · Session · Rerank"] - C --> D["lloyal.node
SessionContext · binaries · GPU selection"] - D --> E["liblloyal
C++20 kernel — the tree ops"] + A["@lloyal-labs/rig
Apps · retrieval"] --> B["@lloyal-labs/lloyal-agents
agents · pools · spines"] + B --> C["@lloyal-labs/sdk
Branch · BranchStore
Session · Rerank"] + C --> D["lloyal.node
SessionContext
binaries · GPU choice"] + D --> E["liblloyal
C++20 kernel · tree ops"] E --> F["llama.cpp b9581"] ``` From 0afffebed302cee54f910584bf378d3089c55611 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:44:48 +1000 Subject: [PATCH 24/54] =?UTF-8?q?docs(readme):=20Abilities,=20not=20Apps?= =?UTF-8?q?=20=E2=80=94=20and=20retrieval=20is=20part=20of=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified against rig rather than renamed by search: defineAbility has 29 references and defineApp has zero, so the rename has landed. The context and type names moved with it — AbilityRegistryCtx, AbilityConfigStoreCtx, AbilityManifest — and the README still named the old ones, which would not resolve. Retrieval is no longer listed as a peer of Abilities. rig's sources/ and reranker are what an Ability is built from, not a separate product surface beside it. Left the licence paragraph alone: "HDK App distribution channel" is legal text, and App there is the other sense — the end-user harness, not the signed capability package. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8bfd30e..f9e0333 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk ```mermaid flowchart TD - A["@lloyal-labs/rig
Apps · retrieval"] --> B["@lloyal-labs/lloyal-agents
agents · pools · spines"] + A["@lloyal-labs/rig
Abilities · tools"] --> B["@lloyal-labs/lloyal-agents
agents · pools · spines"] B --> C["@lloyal-labs/sdk
Branch · BranchStore
Session · Rerank"] C --> D["lloyal.node
SessionContext
binaries · GPU choice"] D --> E["liblloyal
C++20 kernel · tree ops"] @@ -99,7 +99,7 @@ Everything above `lloyal.node` is backend-agnostic; everything below is native. - `loadBinary(variant?)` — pick the variant explicitly - The prebuilt binaries themselves -**Re-exported, so one install is enough:** `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics — and from the agents package `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, plus the App protocol surfaces that pair with rig's `defineApp`. +**Re-exported, so one install is enough:** `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics — and from the agents package `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, plus the Ability protocol surfaces (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`) that pair with rig's `defineAbility` / `createAbilityRegistry`. ## The native surface @@ -214,9 +214,9 @@ Multimodal runs two tiers: SmolVLM-256M for plain positions in CI, Qwen3.5-4B + | Package | Description | | --- | --- | | [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives | -| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + App protocol primitives | -| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | App helpers, retrieval providers, framework tools | -| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses and Apps, publish/install signed Apps | +| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + Ability protocol primitives | +| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | Abilities — `defineAbility`, the registry, and the retrieval and framework tools they are built from | +| [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses and Abilities, publish/install signed Abilities | | [liblloyal](https://github.com/lloyal-ai/liblloyal) | The C++20 kernel | | **lloyal.node** | This package — native backend + prebuilt binaries | | [nitro-llama](https://github.com/lloyal-ai/nitro-llama) | React Native backend via Nitro Modules | From e7741f0bdad501efdd92967f1840c2e3a170c79e Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:55:48 +1000 Subject: [PATCH 25/54] docs(readme): document the backend pack, and why Blackwell needs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolution table referenced "a cached backend pack" twice without ever saying what one is or where it comes from. Both rows now link to a section that explains it. The why, which was nowhere in the docs: the npm packages carry native SASS for mainstream architectures, and Blackwell is newer than most published builds — an sm_100 device would fall back to JIT or to CPU. The pack is a signed dynamically-loaded backend bundle from apps.lloyal.ai, cached per lloyal.node version. Documents the three gates as the source implements them — device (real SASS or a JIT-able PTX floor), driver (can it JIT the pack's toolkit PTX), runtime (does the installed CUDA meet the manifest minimum, or is a companion runtime needed) — and what they decide, taken from the pinned unit matrix: B200 recommended on native SASS, H100 offered only where the driver can JIT, L4 never offered because npm already ships native for it. States the consent boundary plainly, since it is the surprising part: loadBinary() USES a verified cache but never creates one. A pack only ever arrives through an explicit ensureBackendPack() or a provisioner. --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f9e0333..e35b230 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,45 @@ Resolution is ordered and mostly invisible — but when the wrong binary loads, | # | Source | If it fails | | --- | --- | --- | | 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | -| 2 | `LLOYAL_BACKEND_DIR` → that pack | throws; asserts the devices you asked for | -| 3 | a cached backend pack | throws if present-but-invalid — **no** fallthrough to npm | +| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#frontier-gpus--the-backend-pack) | throws; asserts the devices you asked for | +| 3 | a cached [backend pack](#frontier-gpus--the-backend-pack) | throws if present-but-invalid — **no** fallthrough to npm | | 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | | 5 | local `build/Release` | continues — fresher than an installed package during development | | 6 | default CPU package for the platform | throws, naming everything it tried | +## Frontier GPUs — the backend pack + +The npm packages carry native SASS for mainstream architectures. **Blackwell is newer than most published builds**, so an sm_100 device would otherwise fall back to JIT or to CPU. The backend pack closes that gap: a signed, dynamically-loaded backend bundle fetched from `apps.lloyal.ai` and cached per lloyal.node version. + +```javascript +import { probeBackendPack, ensureBackendPack } from "@lloyal-labs/lloyal.node"; + +const offer = await probeBackendPack(); // inspects only — never downloads +if (offer.recommended) await ensureBackendPack(); +``` + +**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — the cache appears only through an explicit `ensureBackendPack()` or a provisioner. + +Three gates run before a pack is even offered: + +| Gate | Question | +| --- | --- | +| device | is the GPU covered by real SASS, or by a JIT-able PTX floor? | +| driver | native SASS needs no JIT; otherwise, can the driver JIT the pack's toolkit PTX? | +| runtime | does the installed CUDA runtime meet the manifest's minimum, or is a companion runtime needed too? | + +What that decides in practice: + +| GPU | Outcome | +| --- | --- | +| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion runtime with it | +| H100 (sm_90) | PTX only — offered where the driver can JIT the pack's toolkit | +| L4 (sm_89) | never offered; the npm package already ships native for it | +| no NVIDIA GPU | never offered | + +Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than quietly falling through to npm, which is why it sits above the variant lookup in the table above. + ## Quick start ```javascript From 59061eabbc988460272498b3084c02fd5f8aaf5a Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 20:58:20 +1000 Subject: [PATCH 26/54] docs(readme): the backend pack is not only a GPU story MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answering "does backend-dl do more?" — it does, in three ways the first pass missed. CPU microarchitectures. Built with GGML_CPU_ALL_VARIANTS and gated at >=8 libggml-cpu-.so modules, so the host's best instruction set is chosen at load rather than baked in. Framing the pack as purely frontier-GPU undersold half of what it carries. The companion CUDA runtime is its own archive — cudart, cublas, cublasLt, nvJitLink — so a host with an older CUDA runs the pack without touching its system install. The pack is a full lloyal.node addon plus its backends, not loose libraries, and on load assertRequestedDevices calls listDevices() and throws when a GPU was requested but none registered. A pack that quietly came up CPU-only fails loudly instead of just being slow — worth stating, since silent CPU fallback is the failure people actually hit. Also now states the scope I declined to assert last time, having checked: packs are published for linux-x64 with CUDA, and LLOYAL_BACKEND_DL=1 refuses any other combination at build time. And the per-arch cuobjdump fatbin gate, which is what stops the Blackwell claim rotting — publishing fails if a declared arch lacks its SASS or PTX. --- README.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e35b230..cf8e298 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,22 @@ Resolution is ordered and mostly invisible — but when the wrong binary loads, | # | Source | If it fails | | --- | --- | --- | | 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | -| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#frontier-gpus--the-backend-pack) | throws; asserts the devices you asked for | -| 3 | a cached [backend pack](#frontier-gpus--the-backend-pack) | throws if present-but-invalid — **no** fallthrough to npm | +| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws; asserts the devices you asked for | +| 3 | a cached [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws if present-but-invalid — **no** fallthrough to npm | | 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | | 5 | local `build/Release` | continues — fresher than an installed package during development | | 6 | default CPU package for the platform | throws, naming everything it tried | -## Frontier GPUs — the backend pack +## The backend pack — frontier GPUs, and every CPU -The npm packages carry native SASS for mainstream architectures. **Blackwell is newer than most published builds**, so an sm_100 device would otherwise fall back to JIT or to CPU. The backend pack closes that gap: a signed, dynamically-loaded backend bundle fetched from `apps.lloyal.ai` and cached per lloyal.node version. +The npm packages are one build per platform/GPU pair. The backend pack is the other shape: **one artifact carrying many backends as separately loadable modules** (`GGML_BACKEND_DL`), chosen at load time. It is a full `lloyal.node` addon plus its backends, not a set of loose libraries. + +That buys two things. + +**Frontier GPUs.** Blackwell is newer than most published builds, so an **sm_100** device would otherwise fall back to JIT or to CPU. The pack ships real SASS for it, and a per-arch `cuobjdump` gate at publish time refuses to build one where any declared arch is missing its SASS or PTX — the claim cannot silently rot. + +**Every CPU microarchitecture.** Built with `GGML_CPU_ALL_VARIANTS`, gated at **≥8** `libggml-cpu-.so` modules, so the best instruction set for the host is picked at load rather than baked in. This is not only a GPU feature. ```javascript import { probeBackendPack, ensureBackendPack } from "@lloyal-labs/lloyal.node"; @@ -52,26 +58,28 @@ const offer = await probeBackendPack(); // inspects only — never downlo if (offer.recommended) await ensureBackendPack(); ``` -**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — the cache appears only through an explicit `ensureBackendPack()` or a provisioner. +**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — a pack arrives only through an explicit `ensureBackendPack()` or a provisioner. On load it also calls `listDevices()` and **throws if a GPU was requested and none registered**, so a pack that quietly came up CPU-only fails loudly instead of being slow. -Three gates run before a pack is even offered: +Three gates run before a pack is offered at all: | Gate | Question | | --- | --- | | device | is the GPU covered by real SASS, or by a JIT-able PTX floor? | | driver | native SASS needs no JIT; otherwise, can the driver JIT the pack's toolkit PTX? | -| runtime | does the installed CUDA runtime meet the manifest's minimum, or is a companion runtime needed too? | - -What that decides in practice: +| runtime | does the installed CUDA runtime meet the manifest's minimum, or is the companion runtime needed too? | | GPU | Outcome | | --- | --- | -| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion runtime with it | -| H100 (sm_90) | PTX only — offered where the driver can JIT the pack's toolkit | +| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion archive with it | +| H100 (sm_90) | PTX only — offered where the driver can JIT | | L4 (sm_89) | never offered; the npm package already ships native for it | | no NVIDIA GPU | never offered | -Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than quietly falling through to npm, which is why it sits above the variant lookup in the table above. +The companion runtime is its own archive — `cudart`, `cublas`, `cublasLt`, `nvJitLink` — so a host with an older CUDA can still run the pack without touching its system install. + +Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than falling through to npm, which is why it sits above the variant lookup in the table above. + +> Packs are published for **linux-x64 with CUDA** today; `LLOYAL_BACKEND_DL=1` refuses any other combination at build time. Everywhere else, the npm packages are the whole story. ## Quick start From 4d57ee0eccec28885c7d14a768236190aed6995c Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:00:13 +1000 Subject: [PATCH 27/54] docs(readme): separate the two distribution channels explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pack section could be read as 'lloyal.node is linux-only for CUDA', which is wrong and is exactly the misreading it produced. Windows CUDA ships as one of the 13 prebuilt npm packages (win32-x64-cuda, release.yml:68) and needs nothing from the pack. The pack is a separate opt-in channel for linux-x64 alone — BackendPackPlatform is the single literal 'linux-x64-dl' and platformTag() returns null everywhere else, so it is never looked for off that platform. linux-arm64 is the named follow-on; Windows is not. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf8e298..4f20859 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,9 @@ The companion runtime is its own archive — `cudart`, `cublas`, `cublasLt`, `nv Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than falling through to npm, which is why it sits above the variant lookup in the table above. -> Packs are published for **linux-x64 with CUDA** today; `LLOYAL_BACKEND_DL=1` refuses any other combination at build time. Everywhere else, the npm packages are the whole story. +> **Two channels, not one.** The 13 prebuilt npm packages cover macOS, Linux **and Windows** — `win32-x64-cuda` is one of them, so Windows CUDA ships that way and needs nothing from this section. +> +> The backend pack is a separate, opt-in channel published only for **linux-x64**. `platformTag()` returns `null` anywhere else, so a pack is never even looked for, and `LLOYAL_BACKEND_DL=1` refuses to build one. linux-arm64 is the named follow-on. ## Quick start From 727532bb7463a7940752d561a5b67f284068d0dc Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:05:54 +1000 Subject: [PATCH 28/54] docs(readme): correct a re-export I invented, and attribute grants properly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Ability protocol surfaces are NOT re-exported from lloyal.node — grep for "Ability" in src/index.ts returns nothing. I asserted they were during the Abilities rename, so anyone following that line would have written an import that fails. The re-export list is now what src/index.ts actually exports, and a second line names what must come from its own package. Grants: rig does not own them. agents owns the contract — interface GrantStore (grant-store.ts:41), GrantStoreCtx (context.ts:164) and the authGuard that reads it — deliberately, so the framework context and ability code share it without a dependency cycle. rig ships createGrantStore, the in-memory reference store for dev harnesses, examples and tests; a harness backing grants with a secrets manager implements the interface itself. Both rows now say which half they hold. Ordering was already right and stays: rig depends on @lloyal-labs/lloyal-agents (and sdk, lloyal.node, channel-verify), agents depends only on sdk, so rig → agents → sdk is the real direction in the layer diagram, and the ecosystem table already lists agents ahead of rig. --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4f20859..b63ce2e 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,9 @@ Everything above `lloyal.node` is backend-agnostic; everything below is native. - `loadBinary(variant?)` — pick the variant explicitly - The prebuilt binaries themselves -**Re-exported, so one install is enough:** `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics — and from the agents package `useAgent`, `agentPool`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, plus the Ability protocol surfaces (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`) that pair with rig's `defineAbility` / `createAbilityRegistry`. +**Re-exported, so one install is enough:** from the SDK, `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar` and the per-token metrics; from agents, `Tool`, `Agent`, `agent`, `agentPool`, `useAgent`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, `initAgents`, `DefaultAgentPolicy`, `renderTemplate`. + +**Not re-exported** — import these from their own packages: the Ability protocol (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`, and `GrantStoreCtx`) from `@lloyal-labs/lloyal-agents`, and `defineAbility` / `createAbilityRegistry` / `createGrantStore` from `@lloyal-labs/rig`. ## The native surface @@ -256,8 +258,8 @@ Multimodal runs two tiers: SmolVLM-256M for plain positions in CI, Qwen3.5-4B + | Package | Description | | --- | --- | | [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) | Backend-agnostic inference primitives | -| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime + Ability protocol primitives | -| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | Abilities — `defineAbility`, the registry, and the retrieval and framework tools they are built from | +| [`@lloyal-labs/lloyal-agents`](https://github.com/lloyal-ai/hdk/tree/main/packages/agents) | Multi-agent runtime; owns the Ability protocol contracts and the `GrantStore` / `authGuard` surface that gates `protected` tools | +| [`@lloyal-labs/rig`](https://github.com/lloyal-ai/hdk/tree/main/packages/rig) | Builds Abilities on those contracts — `defineAbility`, `createAbilityRegistry`, retrieval and framework tools, and `createGrantStore`, the reference in-memory grant store | | [`harness.dev`](https://www.npmjs.com/package/harness.dev) | CLI — scaffold harnesses and Abilities, publish/install signed Abilities | | [liblloyal](https://github.com/lloyal-ai/liblloyal) | The C++20 kernel | | **lloyal.node** | This package — native backend + prebuilt binaries | From bc6006b959ba0e484d6e4d773b75c9e25ec6f61a Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:08:55 +1000 Subject: [PATCH 29/54] docs(readme): lead with code, not deployment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quick start sat behind the binary-resolution table and the whole backend-pack section — a reader landing here got two screens of deployment detail before seeing a single call. Now: Install, Quick start, Who owns what, The native surface, Multimodal, then the deployment group (Which binary loads, GPU variant selection, the backend pack) together at the end where someone shipping to a specific target will look for it. --- README.md | 112 +++++++++++++++++++++++++++--------------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index b63ce2e..97de740 100644 --- a/README.md +++ b/README.md @@ -27,62 +27,6 @@ npm install @lloyal-labs/lloyal.node | Windows | x64 | CPU / CUDA / Vulkan | | Windows | arm64 | CPU / Vulkan | -## Which binary loads - -Resolution is ordered and mostly invisible — but when the wrong binary loads, this is the order that decided it. - -| # | Source | If it fails | -| --- | --- | --- | -| 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | -| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws; asserts the devices you asked for | -| 3 | a cached [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws if present-but-invalid — **no** fallthrough to npm | -| 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | -| 5 | local `build/Release` | continues — fresher than an installed package during development | -| 6 | default CPU package for the platform | throws, naming everything it tried | - - -## The backend pack — frontier GPUs, and every CPU - -The npm packages are one build per platform/GPU pair. The backend pack is the other shape: **one artifact carrying many backends as separately loadable modules** (`GGML_BACKEND_DL`), chosen at load time. It is a full `lloyal.node` addon plus its backends, not a set of loose libraries. - -That buys two things. - -**Frontier GPUs.** Blackwell is newer than most published builds, so an **sm_100** device would otherwise fall back to JIT or to CPU. The pack ships real SASS for it, and a per-arch `cuobjdump` gate at publish time refuses to build one where any declared arch is missing its SASS or PTX — the claim cannot silently rot. - -**Every CPU microarchitecture.** Built with `GGML_CPU_ALL_VARIANTS`, gated at **≥8** `libggml-cpu-.so` modules, so the best instruction set for the host is picked at load rather than baked in. This is not only a GPU feature. - -```javascript -import { probeBackendPack, ensureBackendPack } from "@lloyal-labs/lloyal.node"; - -const offer = await probeBackendPack(); // inspects only — never downloads -if (offer.recommended) await ensureBackendPack(); -``` - -**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — a pack arrives only through an explicit `ensureBackendPack()` or a provisioner. On load it also calls `listDevices()` and **throws if a GPU was requested and none registered**, so a pack that quietly came up CPU-only fails loudly instead of being slow. - -Three gates run before a pack is offered at all: - -| Gate | Question | -| --- | --- | -| device | is the GPU covered by real SASS, or by a JIT-able PTX floor? | -| driver | native SASS needs no JIT; otherwise, can the driver JIT the pack's toolkit PTX? | -| runtime | does the installed CUDA runtime meet the manifest's minimum, or is the companion runtime needed too? | - -| GPU | Outcome | -| --- | --- | -| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion archive with it | -| H100 (sm_90) | PTX only — offered where the driver can JIT | -| L4 (sm_89) | never offered; the npm package already ships native for it | -| no NVIDIA GPU | never offered | - -The companion runtime is its own archive — `cudart`, `cublas`, `cublasLt`, `nvJitLink` — so a host with an older CUDA can still run the pack without touching its system install. - -Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than falling through to npm, which is why it sits above the variant lookup in the table above. - -> **Two channels, not one.** The 13 prebuilt npm packages cover macOS, Linux **and Windows** — `win32-x64-cuda` is one of them, so Windows CUDA ships that way and needs nothing from this section. -> -> The backend pack is a separate, opt-in channel published only for **linux-x64**. `platformTag()` returns `null` anywhere else, so a pack is never even looked for, and `LLOYAL_BACKEND_DL=1` refuses to build one. linux-arm64 is the named follow-on. - ## Quick start ```javascript @@ -212,6 +156,20 @@ const [{ tokensDecoded, positionAdvance }] = A configured `mmprojPath` that fails to load throws at `createContext` — never a silent fall back to text-only. Audio is rejected explicitly. +## Which binary loads + +Resolution is ordered and mostly invisible — but when the wrong binary loads, this is the order that decided it. + +| # | Source | If it fails | +| --- | --- | --- | +| 1 | `LLOYAL_LOCAL=1` → `build/Release` | **throws** — never falls back to a published binary | +| 2 | `LLOYAL_BACKEND_DIR` → that [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws; asserts the devices you asked for | +| 3 | a cached [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) | throws if present-but-invalid — **no** fallthrough to npm | +| 4 | requested variant — `loadBinary()` argument or `LLOYAL_GPU` | warns and continues, unless `LLOYAL_NO_FALLBACK=1` | +| 5 | local `build/Release` | continues — fresher than an installed package during development | +| 6 | default CPU package for the platform | throws, naming everything it tried | + + ## GPU variant selection ```javascript @@ -226,6 +184,48 @@ const ctx2 = await binding.createContext({ modelPath: "./model.gguf" }); // Falls back to CPU with a warning unless LLOYAL_NO_FALLBACK=1 ``` +## The backend pack — frontier GPUs, and every CPU + +The npm packages are one build per platform/GPU pair. The backend pack is the other shape: **one artifact carrying many backends as separately loadable modules** (`GGML_BACKEND_DL`), chosen at load time. It is a full `lloyal.node` addon plus its backends, not a set of loose libraries. + +That buys two things. + +**Frontier GPUs.** Blackwell is newer than most published builds, so an **sm_100** device would otherwise fall back to JIT or to CPU. The pack ships real SASS for it, and a per-arch `cuobjdump` gate at publish time refuses to build one where any declared arch is missing its SASS or PTX — the claim cannot silently rot. + +**Every CPU microarchitecture.** Built with `GGML_CPU_ALL_VARIANTS`, gated at **≥8** `libggml-cpu-.so` modules, so the best instruction set for the host is picked at load rather than baked in. This is not only a GPU feature. + +```javascript +import { probeBackendPack, ensureBackendPack } from "@lloyal-labs/lloyal.node"; + +const offer = await probeBackendPack(); // inspects only — never downloads +if (offer.recommended) await ensureBackendPack(); +``` + +**Nothing is fetched without consent.** `loadBinary()` will *use* a verified cache if one exists, but never creates one — a pack arrives only through an explicit `ensureBackendPack()` or a provisioner. On load it also calls `listDevices()` and **throws if a GPU was requested and none registered**, so a pack that quietly came up CPU-only fails loudly instead of being slow. + +Three gates run before a pack is offered at all: + +| Gate | Question | +| --- | --- | +| device | is the GPU covered by real SASS, or by a JIT-able PTX floor? | +| driver | native SASS needs no JIT; otherwise, can the driver JIT the pack's toolkit PTX? | +| runtime | does the installed CUDA runtime meet the manifest's minimum, or is the companion runtime needed too? | + +| GPU | Outcome | +| --- | --- | +| **B200** (sm_100, Blackwell) | native SASS → **recommended**; an older CUDA runtime pulls the companion archive with it | +| H100 (sm_90) | PTX only — offered where the driver can JIT | +| L4 (sm_89) | never offered; the npm package already ships native for it | +| no NVIDIA GPU | never offered | + +The companion runtime is its own archive — `cudart`, `cublas`, `cublasLt`, `nvJitLink` — so a host with an older CUDA can still run the pack without touching its system install. + +Then download → verify (sha256 plus the platform signature on the manifest) → extract → cache. A present-but-invalid cache **throws** rather than falling through to npm, which is why it sits above the variant lookup in the table above. + +> **Two channels, not one.** The 13 prebuilt npm packages cover macOS, Linux **and Windows** — `win32-x64-cuda` is one of them, so Windows CUDA ships that way and needs nothing from this section. +> +> The backend pack is a separate, opt-in channel published only for **linux-x64**. `platformTag()` returns `null` anywhere else, so a pack is never even looked for, and `LLOYAL_BACKEND_DL=1` refuses to build one. linux-arm64 is the named follow-on. + ## Examples | Example | Pattern | From 97422ca9d25f0410c21b251ae6718fbc3c667d50 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:16:09 +1000 Subject: [PATCH 30/54] docs(readme): draw the stack as the developer meets it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diagram was a dependency graph wearing a mental-model heading. rig sat on top because rig imports agents — true, and useless to a reader, who never encounters the ecosystem in link order. Now it reads as the path you actually take: you write agents, Abilities plug into them, and everything from the SDK down is machinery you rarely touch. rig hangs off agents on a dotted "plugs into" edge rather than looming above it, because installing an Ability is a thing you add, not a layer you pass through. Dropped the footnote about dependencies running the other way — a README is for the mental model, and package.json already holds the mechanical order. Labels kept to 26 characters per line, the width that renders unclipped. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 97de740..b56eba4 100644 --- a/README.md +++ b/README.md @@ -69,14 +69,14 @@ See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk ```mermaid flowchart TD - A["@lloyal-labs/rig
Abilities · tools"] --> B["@lloyal-labs/lloyal-agents
agents · pools · spines"] - B --> C["@lloyal-labs/sdk
Branch · BranchStore
Session · Rerank"] - C --> D["lloyal.node
SessionContext
binaries · GPU choice"] + B["@lloyal-labs/lloyal-agents
what you write
useAgent · agentPool"] --> C["@lloyal-labs/sdk
Branch · BranchStore
Session · Rerank"] + R["@lloyal-labs/rig
Abilities you install"] -. plugs into .-> B + C --> D["lloyal.node
SessionContext · binaries"] D --> E["liblloyal
C++20 kernel · tree ops"] E --> F["llama.cpp b9581"] ``` -Everything above `lloyal.node` is backend-agnostic; everything below is native. That seam is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. +You write agents; Abilities plug into them; everything from the SDK down is machinery you rarely touch. Everything above `lloyal.node` is backend-agnostic and everything below is native — that seam is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. **Native-only, not in the SDK:** From 0bc12266fdfc5b03dd5bbb950362a439fcd94605 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:20:01 +1000 Subject: [PATCH 31/54] =?UTF-8?q?docs(readme):=20stay=20in=20this=20packag?= =?UTF-8?q?e's=20lane=20=E2=80=94=20it=20owns=20no=20agents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section drew @lloyal-labs/lloyal-agents, sdk and rig as layers of this package. They are not: they live in the hdk repo, and lloyal.node contains zero agent code — every "agent" occurrence in src/ is a name inside a re-export block. Same mistake caught in liblloyal, one layer up. Renaming it to "The stack" would only have relabelled the problem. The diagram was forcing an ordering argument about packages this README has no business ordering, so it is gone. What lloyal.node actually is takes one sentence: it binds liblloyal to Node and ships it prebuilt, and it is the seam between backend-agnostic TypeScript and native code. What remains is this package's surface, split by who owns it: what lloyal.node owns, what it re-exports as a convenience (marked as HDK packages documented there), and what it does not re-export. The ecosystem table already links the HDK packages for anyone who wants them. --- README.md | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b56eba4..aa57845 100644 --- a/README.md +++ b/README.md @@ -65,29 +65,23 @@ for await (const { token, text } of branch) process.stdout.write(text); See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk) for the Branch API, continuous tree batching, KV tenancy and topology. -## Who owns what - -```mermaid -flowchart TD - B["@lloyal-labs/lloyal-agents
what you write
useAgent · agentPool"] --> C["@lloyal-labs/sdk
Branch · BranchStore
Session · Rerank"] - R["@lloyal-labs/rig
Abilities you install"] -. plugs into .-> B - C --> D["lloyal.node
SessionContext · binaries"] - D --> E["liblloyal
C++20 kernel · tree ops"] - E --> F["llama.cpp b9581"] -``` +## What this package is -You write agents; Abilities plug into them; everything from the SDK down is machinery you rarely touch. Everything above `lloyal.node` is backend-agnostic and everything below is native — that seam is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. +lloyal.node binds [liblloyal](https://github.com/lloyal-ai/liblloyal) — the C++20 kernel, on llama.cpp b9581 — to Node, and ships it prebuilt. It is the seam: everything above it is backend-agnostic TypeScript, everything below is native. That is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. -**Native-only, not in the SDK:** +**What it owns:** - `createContext(options)` — load a GGUF, get a `SessionContext`. `mmprojPath` loads a multimodal projector beside it. - `_storePrefillMultimodal(...)` — image + text into a branch's KV, plus `supportsVision()` / `supportsAudio()` -- `loadBinary(variant?)` — pick the variant explicitly -- The prebuilt binaries themselves +- `loadBinary(variant?)` and the [binary resolution order](#which-binary-loads) +- The prebuilt binaries and the [backend pack](#the-backend-pack--frontier-gpus-and-every-cpu) + +**What it re-exports**, so one install is enough — these are [HDK](https://github.com/lloyal-ai/hdk) packages, documented there: -**Re-exported, so one install is enough:** from the SDK, `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar` and the per-token metrics; from agents, `Tool`, `Agent`, `agent`, `agentPool`, `useAgent`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, `initAgents`, `DefaultAgentPolicy`, `renderTemplate`. +- from `@lloyal-labs/sdk`: `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics +- from `@lloyal-labs/lloyal-agents`: `Tool`, `Agent`, `agent`, `agentPool`, `useAgent`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, `initAgents`, `DefaultAgentPolicy`, `renderTemplate` -**Not re-exported** — import these from their own packages: the Ability protocol (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`, and `GrantStoreCtx`) from `@lloyal-labs/lloyal-agents`, and `defineAbility` / `createAbilityRegistry` / `createGrantStore` from `@lloyal-labs/rig`. +Not re-exported — import from the package itself: the Ability protocol (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`, `GrantStoreCtx`) from `@lloyal-labs/lloyal-agents`, and `defineAbility` / `createAbilityRegistry` / `createGrantStore` from `@lloyal-labs/rig`. ## The native surface From 242955f1296566fb3887d7e254ff6aa5fbc235f3 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 21:24:07 +1000 Subject: [PATCH 32/54] docs(readme): stop restating the llama.cpp pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as liblloyal, plus the sentence I added last commit — "on llama.cpp b9581" put a third copy of a moving number into prose, which would have gone stale on the next sync-llama-cpp bump and made every upstream pull a README commit too. Badge now links to liblloyal/.llama-cpp-version, the file that actually holds the pin, and says "pinned" instead of naming a build. No hardcoded ref left in either README. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa57845..0bcc5bc 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GPU Tests](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml/badge.svg)](https://github.com/lloyal-ai/lloyal.node/actions/workflows/gpu-test.yml) [![npm](https://img.shields.io/npm/v/@lloyal-labs/lloyal.node.svg)](https://www.npmjs.com/package/@lloyal-labs/lloyal.node) [![License](https://img.shields.io/badge/license-FSL--1.1--Apache--2.0-blue.svg)](LICENSE) -[![llama.cpp](https://img.shields.io/badge/llama.cpp-b9581-green.svg)](https://github.com/ggml-org/llama.cpp/releases/tag/b9581) +[![llama.cpp](https://img.shields.io/badge/llama.cpp-pinned-green.svg)](./liblloyal/.llama-cpp-version) **Vertical Inference on Node — the kernel prebuilt for 13 targets, GPU chosen at run time** @@ -67,7 +67,7 @@ See [`@lloyal-labs/sdk`](https://github.com/lloyal-ai/hdk/tree/main/packages/sdk ## What this package is -lloyal.node binds [liblloyal](https://github.com/lloyal-ai/liblloyal) — the C++20 kernel, on llama.cpp b9581 — to Node, and ships it prebuilt. It is the seam: everything above it is backend-agnostic TypeScript, everything below is native. That is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. +lloyal.node binds [liblloyal](https://github.com/lloyal-ai/liblloyal) — the C++20 kernel — to Node, and ships it prebuilt. It is the seam: everything above it is backend-agnostic TypeScript, everything below is native. That is why [nitro-llama](https://github.com/lloyal-ai/nitro-llama) can serve React Native from the same kernel. **What it owns:** From 7de12a5ccff652830a5e639bb4506ba1d3fe9219 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Sun, 30 Aug 2026 22:02:26 +1000 Subject: [PATCH 33/54] =?UTF-8?q?docs:=20say=20what=20this=20package=20is?= =?UTF-8?q?=20=E2=80=94=20the=20HDK's=20Node=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The intro led with packaging and buried what the thing actually is. Worse, package.json — the line npm renders — read "Node.js client for liblloyal+llama.cpp", which bills llama.cpp alongside our kernel and says nothing about what the package does. Both now state the fact: this is the Node runtime for the HDK, built on liblloyal and llama.cpp. Expanded rather than compressed, because the old block put five ideas in one sentence and was unreadable at a glance. Four short paragraphs instead: what the HDK packages are and why they need something native underneath; what createContext returns and what sits below it; that it ships prebuilt for 13 targets with the variant chosen at process start; and that the HDK is re-exported so one install is enough. liblloyal is described by what it enables — Git-like tree ops for live inference state — rather than by its internals, matching its own README. llama.cpp is named for what it does here, model execution, and is no longer the first thing a reader meets. --- README.md | 10 +++++++--- package.json | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0bcc5bc..802bbd2 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,15 @@ [![License](https://img.shields.io/badge/license-FSL--1.1--Apache--2.0-blue.svg)](LICENSE) [![llama.cpp](https://img.shields.io/badge/llama.cpp-pinned-green.svg)](./liblloyal/.llama-cpp-version) -**Vertical Inference on Node — the kernel prebuilt for 13 targets, GPU chosen at run time** +**The Node runtime for the HDK — built on liblloyal and llama.cpp** -[liblloyal](https://github.com/lloyal-ai/liblloyal) is the C++20 kernel: Git-like tree ops over live inference state. This package is how you run it. One `npm install` gets a binary compiled for your platform, a `SessionContext` bound to it, and the rest of the HDK re-exported — so `import { Branch, useAgent } from "@lloyal-labs/lloyal.node"` works without a second package. +The HDK's packages are backend-agnostic TypeScript: `Branch` and `BranchStore` in the SDK, agents and pools above them. They describe what to do with inference state; they cannot decode a token on their own. On Node, this package is what they run on. -Nothing compiles on install. The variant that matches your hardware is chosen when the process starts, so the same artifact ships to a CPU laptop and a CUDA box. +`createContext()` returns the `SessionContext` every HDK primitive drives — forking a branch, batching N branches into one dispatch, prefilling text or image rows into a branch's KV. Underneath it is [liblloyal](https://github.com/lloyal-ai/liblloyal), the header-only C++20 kernel that enables Git-like tree ops for live inference state, on llama.cpp for the model execution. + +It ships **prebuilt for 13 platform/GPU targets**. Nothing compiles on install, and the variant matching your hardware is chosen when the process starts — so the same artifact ships to a laptop and a CUDA box. + +The rest of the HDK is re-exported, so `import { Branch, useAgent } from "@lloyal-labs/lloyal.node"` needs no second package. ## Install diff --git a/package.json b/package.json index f79da1e..cd331a3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@lloyal-labs/lloyal.node", "version": "3.1.1", - "description": "Node.js client for liblloyal+llama.cpp", + "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", "gypfile": false, From ba1dfbd3f519b86e4ed67ae12db09a3530e7eccd Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Tue, 1 Sep 2026 13:40:17 +1000 Subject: [PATCH 34/54] feat(mtmd): measure an image's cost, and isolate a bad one per entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admission could not account for media: an image was the one input that could enter KV ungated. `_cellsMultimodal` answers what a delta will cost before it is decoded — it constructs an `MtmdSource`, reads `cells()`, and discards it, so the price is bitmap decode plus `mtmd_tokenize`, never the clip encode. Measuring is required rather than estimating, because image cost is NOT additive: on Qwen3.5 one and two images both cost 580 cells, three and four both 1142, five 1704. A per-image estimate over-commits ~2x on even counts. `StorePrefillMultimodalWorker` now reports failure PER ENTRY instead of rejecting the whole call. A cohort carries N independent branches: rejecting loses which of them landed, and six agents settling images must not lose five because one page was corrupt. The caller needs per-entry outcomes to prune exactly the poisoned branches. Only a failure outside the loop — an allocation — still rejects, because then there are no per-entry results to report. `MtmdSource` still refuses a marker/bitmap mismatch in its CONSTRUCTOR, before any decode and with the branch untouched, so a bad entry is rejected just as hard; only the reporting channel changed. The integration test follows that contract: it asserts the per-entry error and that a refused entry decodes nothing. The submodule pointer moves to the liblloyal commit this builds against — it recorded an older one while the checkout carried the `cells()` kernel, so CI would have built the binding entry against a header without it. --- liblloyal | 2 +- src/SessionContext.cpp | 130 ++++++++++++++++++++++++++++++++++++++++- src/SessionContext.hpp | 1 + test/integration.ts | 22 +++++-- 4 files changed, 145 insertions(+), 10 deletions(-) diff --git a/liblloyal b/liblloyal index 8859de9..cf6a666 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 8859de929de10c06bcad19dff8705caa3d09b0dd +Subproject commit cf6a666d07416a535eaaacaa4f4572f8d3755d69 diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index d6dc0ef..222f35a 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -695,6 +695,10 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { struct BranchResult { int64_t tokensDecoded = 0; int64_t positionAdvance = 0; + /** Empty when this entry landed. Non-empty ⇒ its branch is POISONED — + * decode_segments is not atomic, and partial-range KV ops are meaningless + * on recurrent layers, so the caller prunes and replays from content. */ + std::string error; }; StorePrefillMultimodalWorker(Napi::Env env, @@ -710,17 +714,28 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { _sepStorage(std::move(sepStorage)), _prompts(std::move(prompts)), _bitmapBytes(std::move(bitmapBytes)) {} + // Per-entry isolation, deliberately: a corrupt image in one agent's tool + // result must not cost its siblings their prefills. Failing the whole call + // would also lose WHICH entries landed, and the caller needs that to prune + // exactly the poisoned branches. Only a failure outside the loop (an + // allocation) can still reject the promise, because then there are no + // per-entry results to report at all. void Execute() override { try { _results.resize(_handles.size()); - for (size_t i = 0; i < _handles.size(); ++i) { + } catch (const std::exception& e) { SetError(e.what()); return; } + + for (size_t i = 0; i < _handles.size(); ++i) { + try { lloyal::MtmdSource source( _mtmd, _prompts[i], _bitmapBytes[i], std::span(_sepStorage[i]), _nEmbdInp); const auto r = _store.decode_segments(_handles[i], source); - _results[i] = { r.cells, static_cast(r.advance) }; + _results[i] = { r.cells, static_cast(r.advance), "" }; + } catch (const std::exception& e) { + _results[i] = { 0, 0, e.what() }; } - } catch (const std::exception& e) { SetError(e.what()); } + } } void OnOK() override { @@ -730,6 +745,9 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { Napi::Object r = Napi::Object::New(env); r.Set("tokensDecoded", Napi::Number::New(env, static_cast(_results[i].tokensDecoded))); r.Set("positionAdvance", Napi::Number::New(env, static_cast(_results[i].positionAdvance))); + if (!_results[i].error.empty()) { + r.Set("error", Napi::String::New(env, _results[i].error)); + } out.Set(static_cast(i), r); } _deferred.Resolve(out); @@ -917,6 +935,7 @@ Napi::Object SessionContext::Init(Napi::Env env, Napi::Object exports) { InstanceMethod("_storeCommit", &SessionContext::_storeCommit), InstanceMethod("_storePrefill", &SessionContext::_storePrefill), InstanceMethod("_storePrefillMultimodal", &SessionContext::_storePrefillMultimodal), + InstanceMethod("_cellsMultimodal", &SessionContext::_cellsMultimodal), InstanceMethod("supportsVision", &SessionContext::supportsVision), InstanceMethod("supportsAudio", &SessionContext::supportsAudio), InstanceMethod("_storeMergeLogits", &SessionContext::_storeMergeLogits), @@ -2610,6 +2629,56 @@ Napi::Value SessionContext::_storePrefill(const Napi::CallbackInfo& info) { return worker->GetPromise(); } +/** + * Reports what a multimodal prefill WOULD cost, without decoding it. + * + * Admission needs a number before the branch is touched: `decode_segments` is + * not atomic, so a caller that discovers the overflow midway has poisoned the + * branch, while one that refuses up front has spent nothing. Text can be + * measured by tokenizing it; an image cannot, because the caller holds bytes + * and the row count depends on the projector's geometry. + * + * Constructing the source is the measurement: `MtmdSource` counts cells after + * `mtmd_tokenize` and BEFORE any clip encode, so this pays bitmap decode plus + * tokenization — not the vision-tower pass, which stays in the prefill. + */ +class CellsMultimodalWorker : public Napi::AsyncWorker { +public: + CellsMultimodalWorker(Napi::Env env, + mtmd_context* mtmd, + int32_t nEmbdInp, + std::vector sep, + std::string prompt, + std::vector> bitmapBytes) + : AsyncWorker(env), _deferred(env), _mtmd(mtmd), _nEmbdInp(nEmbdInp), + _sep(std::move(sep)), _prompt(std::move(prompt)), + _bitmapBytes(std::move(bitmapBytes)) {} + + void Execute() override { + try { + lloyal::MtmdSource source( + _mtmd, _prompt, _bitmapBytes, + std::span(_sep), _nEmbdInp); + _cells = static_cast(source.cells()); + } catch (const std::exception& e) { SetError(e.what()); } + } + + void OnOK() override { + _deferred.Resolve(Napi::Number::New(Env(), static_cast(_cells))); + } + void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + Napi::Promise GetPromise() { return _deferred.Promise(); } + +private: + Napi::Promise::Deferred _deferred; + mtmd_context* _mtmd; + int32_t _nEmbdInp; + std::vector _sep; + std::string _prompt; + std::vector> _bitmapBytes; + int64_t _cells = 0; +}; + Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); @@ -2714,6 +2783,61 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in return worker->GetPromise(); } +Napi::Value SessionContext::_cellsMultimodal(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (!_mtmdContext) { + throw Napi::Error::New(env, + "_cellsMultimodal requires a multimodal context — pass mmprojPath to createContext"); + } + if (info.Length() < 3 || !info[0].IsArray() || !info[1].IsString() || !info[2].IsArray()) { + throw Napi::Error::New(env, + "_cellsMultimodal requires (sepTokens[], prompt, bitmaps[])"); + } + + // One delta at a time: admission is a per-delta question, and SETTLE already + // walks its items in order deducting from a running headroom. + Napi::Array jsSep = info[0].As(); + std::vector sep(jsSep.Length()); + for (uint32_t i = 0; i < jsSep.Length(); i++) { + sep[i] = static_cast(jsSep.Get(i).As().Int32Value()); + } + + std::string prompt = info[1].As().Utf8Value(); + + // Marshal on the JS thread — the worker owns copies (Buffers must never be + // touched off-thread). + Napi::Array jsImgs = info[2].As(); + std::vector> bitmapBytes(jsImgs.Length()); + for (uint32_t j = 0; j < jsImgs.Length(); j++) { + Napi::Value v = jsImgs.Get(j); + // Uint8Array specifically, not any TypedArray: an Int32Array would pass a + // bare IsTypedArray() check and then be reinterpreted as raw bytes. + const bool isU8 = v.IsTypedArray() && + v.As().TypedArrayType() == napi_uint8_array; + if (!v.IsBuffer() && !isU8) { + throw Napi::Error::New(env, "_cellsMultimodal: bitmaps must be Buffer/Uint8Array"); + } + if (v.IsBuffer()) { + auto buf = v.As>(); + bitmapBytes[j].assign(buf.Data(), buf.Data() + buf.Length()); + } else { + auto ta = v.As(); + auto* base = static_cast(ta.ArrayBuffer().Data()) + ta.ByteOffset(); + bitmapBytes[j].assign(base, base + ta.ByteLength()); + } + } + + const int32_t nEmbdInp = llama_model_n_embd_inp(_model.get()); + + auto* worker = new CellsMultimodalWorker( + env, _mtmdContext, nEmbdInp, + std::move(sep), std::move(prompt), std::move(bitmapBytes)); + worker->Queue(); + return worker->GetPromise(); +} + Napi::Value SessionContext::supportsVision(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); ensureNotDisposed(); diff --git a/src/SessionContext.hpp b/src/SessionContext.hpp index 8684d17..4472bed 100644 --- a/src/SessionContext.hpp +++ b/src/SessionContext.hpp @@ -301,6 +301,7 @@ class SessionContext : public Napi::ObjectWrap { * Returns: Promise<{tokensDecoded, positionAdvance}[]> */ Napi::Value _storePrefillMultimodal(const Napi::CallbackInfo& info); + Napi::Value _cellsMultimodal(const Napi::CallbackInfo& info); /** True when the loaded mmproj has a vision encoder (no mmproj → false). */ Napi::Value supportsVision(const Napi::CallbackInfo& info); diff --git a/test/integration.ts b/test/integration.ts index 084d95a..2fe68af 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -2444,15 +2444,25 @@ async function testMultimodal(): Promise { } catch { dupThrew = true; } assert(dupThrew, 'duplicate handle in one _storePrefill rejects'); - // Marker/bitmap count mismatch throws (mtmd_tokenize rc=1 path) - let mismatchThrew = false; - try { + // Marker/bitmap count mismatch is reported PER ENTRY, not by rejecting the + // whole call. A cohort carries N independent branches: rejecting would lose + // which of them landed, and six agents settling images must not lose five + // because one page was corrupt. `MtmdSource` still refuses in its + // CONSTRUCTOR — before any decode, branch untouched — so the entry is + // rejected just as hard; only the reporting channel changed. + let mismatchErr = ''; + { const b = Branch.create(ctx, 0, { temperature: 0 }); try { - await mm._storePrefillMultimodal([b.handle], [[]], [`x ${MEDIA_MARKER}`], [[IMG, IMG]]); + const r = await mm._storePrefillMultimodal( + [b.handle], [[]], [`x ${MEDIA_MARKER}`], [[IMG, IMG]]); + mismatchErr = r[0].error ?? ''; + assert(r[0].tokensDecoded === 0, + 'a refused entry decodes nothing (its branch is untouched)'); } finally { await b.prune(); } - } catch { mismatchThrew = true; } - assert(mismatchThrew, 'marker/bitmap count mismatch rejects'); + } + assert(/marker count/i.test(mismatchErr), + `marker/bitmap mismatch reported per entry: "${mismatchErr}"`); // Assertion 8 — frames (the video-carrying contract): several marker'd // frames + timestamp text in ONE prefill. From 97a66077ebdbe1ad205c334186d14bc9cd1809cf Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Tue, 1 Sep 2026 13:40:35 +1000 Subject: [PATCH 35/54] refuse a second native addon image in one process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two copies of this addon in one process is a segfault waiting for a branch handle. Each image carries its own copy of every C++ static — its own BranchStore, slot deque and freelist — so a handle minted against one and resolved against the other reads a garbage slot index and dies in `allocate_slot`, code that has nothing to do with the mistake. It took a day to find, through a disassembly and an ASan run, because nothing in the stack says the two images exist. It is easy to arrange by accident. Node identifies a module by its REALPATH, so a locally linked `@lloyal-labs/rig` resolves ITS `@lloyal-labs/lloyal.node` from its own tree, not the host project's — and `getBinary`'s memo is module-level, so each instance loads its own binary. A published install never hits it (one flat tree, one hoisted copy), which is why it only ever bites during local development, where it is hardest to attribute. Two detectors, because they cover different load orders: a `globalThis` symbol claim catches a sibling that also carries this guard, and a `require.cache` scan for other `lloyal.node` entries whose exports are not ours catches a copy at ANY version that loaded BEFORE us. Identity is the test, not the path — Node caches an addon by resolved filename, so two module instances requiring the same file share one exports object and are correct. `globalThis` is per-thread, so a worker thread loading its own image is unaffected. Known hole, stated rather than hidden: guarded-first-then-unguarded cannot fire, since nothing of ours runs in the older copy. It closes the class once a guarded version is published. --- src/index.ts | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 95cf5c8..e11a74b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -80,6 +80,80 @@ const tryLoadPackage = ( } }; +/** + * One native addon image per thread — a second one is refused, loudly. + * + * Node identifies a module by its REALPATH, so a symlinked package resolves + * its own dependencies from its own tree rather than the host project's. A + * locally linked `@lloyal-labs/rig` therefore reaches a DIFFERENT copy of + * `@lloyal-labs/lloyal.node` than the project does, and both get dlopen'd + * into one process. + * + * Each image carries its own copy of every C++ static — its own BranchStore, + * its own slot deque, its own freelist. A branch handle minted against one + * image and resolved against the other reads a garbage slot index, and the + * process dies with SIGSEGV or SIGBUS inside `allocate_slot`, code that has + * nothing to do with the mistake. This guard turns that into one startup + * error naming both packages. + * + * Identity is the test, not the path: Node caches a native addon by resolved + * filename, so two module instances requiring the SAME file share one exports + * object and are correct. `globalThis` is per-thread, so a worker thread that + * loads its own image is unaffected. + */ +const IMAGE_CLAIM = Symbol.for("@lloyal-labs/lloyal.node:image"); + +type ImageClaim = { binding: NativeBinding; owner: string }; + +/** + * Addon images already in this thread's require cache that are NOT ours. + * + * The symbol claim below only catches a sibling that also carries this guard. + * A copy old enough to predate it claims nothing, so scan the cache directly: + * a native addon is cached by resolved filename, and the entry whose exports + * ARE our binding is our own. Anything else named lloyal.node is a second + * image. This sees a foreign copy that loaded BEFORE us at any version; one + * that loads after us is caught only once it carries this guard too. + */ +const foreignImages = (binding: NativeBinding): string[] => { + const cache = (require as unknown as { + cache?: Record; + }).cache; + if (!cache) return []; + return Object.keys(cache).filter( + (file) => + path.basename(file) === "lloyal.node" && + cache[file]?.exports !== binding, + ); +}; + +const claimImage = (binding: NativeBinding): NativeBinding => { + const globals = globalThis as unknown as Record< + symbol, + ImageClaim | undefined + >; + const held = globals[IMAGE_CLAIM]; + const foreign = held ? [] : foreignImages(binding); + + if (!held && foreign.length === 0) { + globals[IMAGE_CLAIM] = { binding, owner: __dirname }; + return binding; + } + if (held?.binding === binding) return binding; + + throw new Error( + `[lloyal.node] Two different native addon images in one process.\n` + + ` already loaded: ${held?.owner ?? foreign.join(", ")}\n` + + ` now loading : ${__dirname}\n` + + `Each image has its own BranchStore and freelist, so a branch handle ` + + `that crosses between them faults in allocate_slot. This is almost ` + + `always a local dev link: a symlinked @lloyal-labs package resolves ` + + `lloyal.node from its own node_modules instead of the project's. Point ` + + `every copy at one build, or set LLOYAL_BACKEND_DIR to the directory ` + + `containing lloyal.node.`, + ); +}; + /** * Load native binary for a specific GPU variant * @@ -122,7 +196,9 @@ const tryLoadPackage = ( * * @param variant GPU variant: 'cuda', 'vulkan', or undefined for CPU * @returns Native binary module with createContext method - * @throws Error if no binary available for the current platform + * @throws Error if no binary is available for the current platform, or if a + * DIFFERENT addon image is already loaded in this thread — see {@link + * claimImage} for why a second image is fatal rather than merely wasteful. * * @example * ```typescript @@ -138,7 +214,11 @@ const tryLoadPackage = ( * * @category Core */ -export const loadBinary = (variant?: GpuVariant): NativeBinding => { +export const loadBinary = (variant?: GpuVariant): NativeBinding => + claimImage(resolveBinary(variant)); + +/** The resolution chain documented on {@link loadBinary}. */ +const resolveBinary = (variant?: GpuVariant): NativeBinding => { const resolvedVariant = variant ?? process.env.LLOYAL_GPU; const noFallback = process.env.LLOYAL_NO_FALLBACK === "1"; const useLocal = process.env.LLOYAL_LOCAL === "1"; From 6672ff28ac0269c9cabf71eaa1115afd121f355c Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Tue, 1 Sep 2026 19:36:20 +1000 Subject: [PATCH 36/54] alpha channel: cut 0, and GPU tests are skippable without vetoing publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3.2.0-alpha.0 — the publish job already maps an -alpha version to `npm publish --tag alpha`, so committing the version on the arc branch IS the channel: `latest` cannot move from here. The publish gate now treats deliberately-skipped GPU tests as passing (`skip_gpu` dispatch input) — a skip is not a veto, but a GPU FAILURE still blocks. Alpha cuts run the full 13-target platform matrix either way. --- .github/workflows/release.yml | 15 +++++++++++++-- package-lock.json | 30 +++++++++++++++--------------- package.json | 28 ++++++++++++++-------------- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 65260c2..a74b315 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,10 @@ on: description: 'Skip npm publish (for testing)' type: boolean default: true + skip_gpu: + description: 'Skip GPU tests (alpha cuts — CPU/Metal matrix still gates)' + type: boolean + default: false jobs: build-and-test: @@ -329,6 +333,7 @@ jobs: gpu-tests: name: GPU Tests needs: build-and-test + if: github.event_name == 'push' || inputs.skip_gpu != true uses: ./.github/workflows/gpu-test.yml secrets: inherit permissions: @@ -674,8 +679,14 @@ jobs: name: Publish all packages needs: [build-and-test, gpu-tests] runs-on: ubuntu-latest - # Only run if ALL jobs succeeded AND either it's a tag push or manual run without skip - if: success() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.skip_publish)) + # Publish when the matrix succeeded and GPU tests either succeeded or were + # deliberately skipped (alpha cuts) — `success()` alone would treat the + # skip as a veto. A GPU FAILURE still blocks. + if: >- + !cancelled() && + needs.build-and-test.result == 'success' && + (needs.gpu-tests.result == 'success' || needs.gpu-tests.result == 'skipped') && + (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.skip_publish)) permissions: contents: read id-token: write # Required for npm provenance diff --git a/package-lock.json b/package-lock.json index 71ea16a..425292c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", + "version": "3.2.0-alpha.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", + "version": "3.2.0-alpha.0", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@lloyal-labs/tsampler": "^0.2.0", @@ -27,19 +27,19 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.1.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.0" }, "peerDependencies": { "@lloyal-labs/lloyal-agents": ">=3.0.0", diff --git a/package.json b/package.json index cd331a3..4271838 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.1.1", + "version": "3.2.0-alpha.0", "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -67,19 +67,19 @@ "typescript": "^5.9.3" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.1.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.1.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.0", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.0" }, "engines": { "node": ">=24.0.0" From 6f5007e0892830da4e5b03b5b71428f08494fca2 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Wed, 2 Sep 2026 00:16:08 +1000 Subject: [PATCH 37/54] =?UTF-8?q?feat:=20tokenToBytes=20=E2=80=94=20the=20?= =?UTF-8?q?byte-level=20twin=20of=20tokenToText?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A BPE piece can end mid-character, so per-token string conversion tears multi-byte UTF-8. The SDK now assembles streamed text from bytes at character boundaries; the binding stays a passthrough. Also corrects the BranchResult::error docstring: an entry can fail before its decode ran (branch untouched) or inside decode_segments (poisoned) — the caller cannot tell which, so the contract is uniform: prune and replay from content; pruning an untouched branch is safe. --- src/SessionContext.cpp | 31 ++++++++++++++++++++++++++++--- src/SessionContext.hpp | 1 + 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index 222f35a..3704fff 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -695,9 +695,13 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { struct BranchResult { int64_t tokensDecoded = 0; int64_t positionAdvance = 0; - /** Empty when this entry landed. Non-empty ⇒ its branch is POISONED — - * decode_segments is not atomic, and partial-range KV ops are meaningless - * on recurrent layers, so the caller prunes and replays from content. */ + /** Empty when this entry landed. Non-empty ⇒ the entry FAILED — either + * before its decode ran (bitmap decode, marker/bitmap mismatch: the + * branch is untouched) or inside decode_segments (the branch is + * POISONED — the op is not atomic, and partial-range KV ops are + * meaningless on recurrent layers). The caller cannot tell which from + * here, so the contract is uniform: prune and replay from content — + * pruning an untouched branch is safe. */ std::string error; }; @@ -860,6 +864,7 @@ Napi::Object SessionContext::Init(Napi::Env env, Napi::Object exports) { Napi::Function func = DefineClass(env, "SessionContext", { // ===== CORE ===== InstanceMethod("tokenToText", &SessionContext::tokenToText), + InstanceMethod("tokenToBytes", &SessionContext::tokenToBytes), InstanceMethod("isStopToken", &SessionContext::isStopToken), InstanceMethod("getEogToken", &SessionContext::getEogToken), InstanceMethod("getTurnSeparator", &SessionContext::getTurnSeparator), @@ -1105,6 +1110,26 @@ Napi::Value SessionContext::tokenToText(const Napi::CallbackInfo& info) { return Napi::String::New(env, text); } +Napi::Value SessionContext::tokenToBytes(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + ensureNotDisposed(); + + if (info.Length() < 1 || !info[0].IsNumber()) { + throw Napi::TypeError::New(env, "Expected token ID (number)"); + } + + llama_token token = static_cast(info[0].As().Int32Value()); + + // Byte-level twin of tokenToText. A BPE piece is a byte sequence that can + // end mid-character, so converting one piece to a JS string can tear + // multi-byte UTF-8 into U+FFFD. The SDK assembles streamed text from bytes + // at character boundaries; the binding stays a passthrough. + std::string text = lloyal::tokenizer::detokenize(_model.get(), token, true); + + return Napi::Buffer::Copy( + env, reinterpret_cast(text.data()), text.size()); +} + // ===== EMBEDDING EXTRACTION ===== Napi::Value SessionContext::encode(const Napi::CallbackInfo& info) { diff --git a/src/SessionContext.hpp b/src/SessionContext.hpp index 4472bed..c4535f2 100644 --- a/src/SessionContext.hpp +++ b/src/SessionContext.hpp @@ -118,6 +118,7 @@ class SessionContext : public Napi::ObjectWrap { * Returns: string */ Napi::Value tokenToText(const Napi::CallbackInfo& info); + Napi::Value tokenToBytes(const Napi::CallbackInfo& info); /** * Check if token is a stop token (EOS) From 9b022e5b2d71befda0269b81d6bc83f0931d0097 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Wed, 2 Sep 2026 00:31:04 +1000 Subject: [PATCH 38/54] feat: the llama_decode rc crosses the boundary as data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BranchResult.rc on the multimodal cohort (per-entry, set from DecodeError, surfaced beside `error`); an `rc` property attached to the rejected error object on the token and commit workers (single catch arm, dynamic_cast — no catch-order hazard; OnError sets the property on the JS object it constructs). Feeds the self-healing ladder (hdk docs/self-healing.md): 1/-1 mean the branch is intact, 2/<-1 mean poison. --- src/SessionContext.cpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index 3704fff..27c0d02 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -626,16 +626,25 @@ class StoreCommitWorker : public Napi::AsyncWorker { } // ~Snapshot frees the swapped-out (post-accept) state + // One arm, dynamic_cast for the rc — no catch-order hazard. The typed + // exception never crosses N-API; the rc travels as data (see OnError). + if (const auto* de = dynamic_cast(&e)) _rc = de->rc; SetError(e.what()); } } void OnOK() override { _deferred.Resolve(Env().Undefined()); } - void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + void OnError(const Napi::Error& err) override { + // Attach the rc to the JS error we construct — a property on an object we + // own, set on the JS thread. This is the whole boundary crossing. + if (_rc != 0) err.Value().As().Set("rc", Napi::Number::New(Env(), _rc)); + _deferred.Reject(err.Value()); + } Napi::Promise GetPromise() { return _deferred.Promise(); } private: Napi::Promise::Deferred _deferred; + int32_t _rc = 0; lloyal::branch::BranchStore& _store; std::vector _items; }; @@ -662,15 +671,23 @@ class StorePrefillWorker : public Napi::AsyncWorker { items[i].tokens = _tokenStorage[i]; } _store.decode_scatter(items); - } catch (const std::exception& e) { SetError(e.what()); } + } catch (const std::exception& e) { + // One arm, dynamic_cast for the rc — no catch-order hazard. + if (const auto* de = dynamic_cast(&e)) _rc = de->rc; + SetError(e.what()); + } } void OnOK() override { _deferred.Resolve(Env().Undefined()); } - void OnError(const Napi::Error& err) override { _deferred.Reject(err.Value()); } + void OnError(const Napi::Error& err) override { + if (_rc != 0) err.Value().As().Set("rc", Napi::Number::New(Env(), _rc)); + _deferred.Reject(err.Value()); + } Napi::Promise GetPromise() { return _deferred.Promise(); } private: Napi::Promise::Deferred _deferred; + int32_t _rc = 0; lloyal::branch::BranchStore& _store; std::vector _handles; std::vector> _tokenStorage; @@ -703,6 +720,10 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { * here, so the contract is uniform: prune and replay from content — * pruning an untouched branch is safe. */ std::string error; + /** llama_decode's raw return code when the failure carried one (0 + * otherwise — validation throws never reached llama_decode). The + * caller's classification: 1/-1 restored, 2/<-1 poisoned. */ + int32_t rc = 0; }; StorePrefillMultimodalWorker(Napi::Env env, @@ -737,7 +758,8 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { const auto r = _store.decode_segments(_handles[i], source); _results[i] = { r.cells, static_cast(r.advance), "" }; } catch (const std::exception& e) { - _results[i] = { 0, 0, e.what() }; + const auto* de = dynamic_cast(&e); + _results[i] = { 0, 0, e.what(), de ? de->rc : 0 }; } } } @@ -751,6 +773,7 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { r.Set("positionAdvance", Napi::Number::New(env, static_cast(_results[i].positionAdvance))); if (!_results[i].error.empty()) { r.Set("error", Napi::String::New(env, _results[i].error)); + if (_results[i].rc != 0) r.Set("rc", Napi::Number::New(env, _results[i].rc)); } out.Set(static_cast(i), r); } From 3354c685fa97d4bc9871062a9e8f78adcd7531bc Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Wed, 2 Sep 2026 02:11:31 +1000 Subject: [PATCH 39/54] alpha: cut 1, and the liblloyal pointer catches up to DecodeError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 3.2.0-alpha.1. The recorded submodule pointer still named cf6a666 while the binding catches lloyal::decode::DecodeError (2e6688b/0aaae86) — CI checks out the pointer, so the 13-target build would not compile. Publishing an alpha is shipping; the pointer ships with it. --- liblloyal | 2 +- package-lock.json | 30 +++++++++++++++--------------- package.json | 28 ++++++++++++++-------------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/liblloyal b/liblloyal index cf6a666..0aaae86 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit cf6a666d07416a535eaaacaa4f4572f8d3755d69 +Subproject commit 0aaae866934eebe9ba10a9bd0603b5b60d64e8b0 diff --git a/package-lock.json b/package-lock.json index 425292c..350f9e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.0", + "version": "3.2.0-alpha.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.0", + "version": "3.2.0-alpha.1", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@lloyal-labs/tsampler": "^0.2.0", @@ -27,19 +27,19 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.0" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.1" }, "peerDependencies": { "@lloyal-labs/lloyal-agents": ">=3.0.0", diff --git a/package.json b/package.json index 4271838..2fbc6d9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.0", + "version": "3.2.0-alpha.1", "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -67,19 +67,19 @@ "typescript": "^5.9.3" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.0", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.0" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.1", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.1" }, "engines": { "node": ">=24.0.0" From 118fb3210c4b037c0e214017caeda0ad97d1f8f3 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Thu, 3 Sep 2026 23:27:31 +1000 Subject: [PATCH 40/54] =?UTF-8?q?feat:=20`partial`=20crosses=20the=20bound?= =?UTF-8?q?ary=20beside=20`rc`;=20liblloyal=20=E2=86=92=20dd4667e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel's DecodeError now says whether earlier chunks of a failed decode landed — rc alone cannot, since llama_decode restores only the call it rejects and the chunked paths keep what came before. The two decode workers attach `partial` to the rejected JS error next to `rc` (same catch arm, same OnError crossing), and the multimodal cohort's per-entry result carries it next to its `rc`. The rule the SDK gates on: intact iff rc == 1 && !partial; anything else, prune and replay. Also from the review: the integration test's multimodal result type carries error/rc/partial (build:test did not compile), and the _storePrefillMultimodal docstring names the actual walk — MtmdSource yields segments, BranchStore::decode_segments places them. Submodule pointer: dd4667e (DecodeError{rc, partial}, one logits capture). Addon rebuilt against it; tsc green. --- liblloyal | 2 +- src/SessionContext.cpp | 40 +++++++++++++++++++++++++++------------- src/SessionContext.hpp | 9 ++++++--- test/integration.ts | 5 ++++- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/liblloyal b/liblloyal index 0aaae86..dd4667e 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 0aaae866934eebe9ba10a9bd0603b5b60d64e8b0 +Subproject commit dd4667e1f67008386ffdfdd5db163d2e1e8feb25 diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index 27c0d02..651a515 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -628,7 +628,7 @@ class StoreCommitWorker : public Napi::AsyncWorker { // One arm, dynamic_cast for the rc — no catch-order hazard. The typed // exception never crosses N-API; the rc travels as data (see OnError). - if (const auto* de = dynamic_cast(&e)) _rc = de->rc; + if (const auto* de = dynamic_cast(&e)) { _rc = de->rc; _partial = de->partial; } SetError(e.what()); } } @@ -637,7 +637,11 @@ class StoreCommitWorker : public Napi::AsyncWorker { void OnError(const Napi::Error& err) override { // Attach the rc to the JS error we construct — a property on an object we // own, set on the JS thread. This is the whole boundary crossing. - if (_rc != 0) err.Value().As().Set("rc", Napi::Number::New(Env(), _rc)); + if (_rc != 0) { + auto o = err.Value().As(); + o.Set("rc", Napi::Number::New(Env(), _rc)); + o.Set("partial", Napi::Boolean::New(Env(), _partial)); + } _deferred.Reject(err.Value()); } Napi::Promise GetPromise() { return _deferred.Promise(); } @@ -645,6 +649,7 @@ class StoreCommitWorker : public Napi::AsyncWorker { private: Napi::Promise::Deferred _deferred; int32_t _rc = 0; + bool _partial = false; lloyal::branch::BranchStore& _store; std::vector _items; }; @@ -673,14 +678,18 @@ class StorePrefillWorker : public Napi::AsyncWorker { _store.decode_scatter(items); } catch (const std::exception& e) { // One arm, dynamic_cast for the rc — no catch-order hazard. - if (const auto* de = dynamic_cast(&e)) _rc = de->rc; + if (const auto* de = dynamic_cast(&e)) { _rc = de->rc; _partial = de->partial; } SetError(e.what()); } } void OnOK() override { _deferred.Resolve(Env().Undefined()); } void OnError(const Napi::Error& err) override { - if (_rc != 0) err.Value().As().Set("rc", Napi::Number::New(Env(), _rc)); + if (_rc != 0) { + auto o = err.Value().As(); + o.Set("rc", Napi::Number::New(Env(), _rc)); + o.Set("partial", Napi::Boolean::New(Env(), _partial)); + } _deferred.Reject(err.Value()); } Napi::Promise GetPromise() { return _deferred.Promise(); } @@ -688,6 +697,7 @@ class StorePrefillWorker : public Napi::AsyncWorker { private: Napi::Promise::Deferred _deferred; int32_t _rc = 0; + bool _partial = false; lloyal::branch::BranchStore& _store; std::vector _handles; std::vector> _tokenStorage; @@ -714,16 +724,17 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { int64_t positionAdvance = 0; /** Empty when this entry landed. Non-empty ⇒ the entry FAILED — either * before its decode ran (bitmap decode, marker/bitmap mismatch: the - * branch is untouched) or inside decode_segments (the branch is - * POISONED — the op is not atomic, and partial-range KV ops are - * meaningless on recurrent layers). The caller cannot tell which from - * here, so the contract is uniform: prune and replay from content — - * pruning an untouched branch is safe. */ + * branch is untouched) or inside decode_segments. `rc` and `partial` + * say which case a decode failure is (see DecodeError in liblloyal): + * intact iff rc == 1 && !partial; anything else ⇒ prune and replay from + * content — pruning an untouched branch is safe. */ std::string error; /** llama_decode's raw return code when the failure carried one (0 - * otherwise — validation throws never reached llama_decode). The - * caller's classification: 1/-1 restored, 2/<-1 poisoned. */ + * otherwise — validation throws never reached llama_decode). */ int32_t rc = 0; + /** True when an earlier chunk of this entry landed before the failing + * call — the branch is then not intact even when `rc` restored state. */ + bool partial = false; }; StorePrefillMultimodalWorker(Napi::Env env, @@ -759,7 +770,7 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { _results[i] = { r.cells, static_cast(r.advance), "" }; } catch (const std::exception& e) { const auto* de = dynamic_cast(&e); - _results[i] = { 0, 0, e.what(), de ? de->rc : 0 }; + _results[i] = { 0, 0, e.what(), de ? de->rc : 0, de ? de->partial : false }; } } } @@ -773,7 +784,10 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { r.Set("positionAdvance", Napi::Number::New(env, static_cast(_results[i].positionAdvance))); if (!_results[i].error.empty()) { r.Set("error", Napi::String::New(env, _results[i].error)); - if (_results[i].rc != 0) r.Set("rc", Napi::Number::New(env, _results[i].rc)); + if (_results[i].rc != 0) { + r.Set("rc", Napi::Number::New(env, _results[i].rc)); + r.Set("partial", Napi::Boolean::New(env, _results[i].partial)); + } } out.Set(static_cast(i), r); } diff --git a/src/SessionContext.hpp b/src/SessionContext.hpp index c4535f2..ce002c9 100644 --- a/src/SessionContext.hpp +++ b/src/SessionContext.hpp @@ -295,11 +295,14 @@ class SessionContext : public Napi::ObjectWrap { /** * Multimodal prefill: per-branch sep tokens + templated prompt (with - * media markers) + image bytes. mtmd tokenizes the prompt, the worker - * walks TEXT/IMAGE chunks in order (token rail / embedding rail). + * media markers) + image bytes. The walk itself is liblloyal's: + * `MtmdSource` yields TEXT/IMAGE segments in order and + * `BranchStore::decode_segments` places them (token rail / embedding rail). * Args: (handles: number[], sepTokens: number[][], prompts: string[], * bitmaps: Buffer[][]) - * Returns: Promise<{tokensDecoded, positionAdvance}[]> + * Returns: Promise<{tokensDecoded, positionAdvance, error?, rc?, partial?}[]> + * — per-entry outcomes; a failed entry carries `error`, and when the + * failure came from llama_decode, its `rc` and `partial` (see DecodeError). */ Napi::Value _storePrefillMultimodal(const Napi::CallbackInfo& info); Napi::Value _cellsMultimodal(const Napi::CallbackInfo& info); diff --git a/test/integration.ts b/test/integration.ts index 2fe68af..1ef8147 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -2304,7 +2304,10 @@ async function testMultimodal(): Promise { const mm = ctx as unknown as { _storePrefillMultimodal( handles: number[], seps: number[][], prompts: string[], bitmaps: Buffer[][], - ): Promise>; + ): Promise>; _storePrefill(handles: number[], tokenArrays: number[][]): Promise; _storeKvPressure(): { cellsUsed: number }; supportsVision(): boolean; From 6962f5cb5b57f523e1271a2ae9c244898c7f8815 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 01:30:14 +1000 Subject: [PATCH 41/54] =?UTF-8?q?test:=20`partial`=20witnessed=20from=20Ja?= =?UTF-8?q?vaScript;=20the=20duplicate=20guard=20coerces=20like=20the=20ma?= =?UTF-8?q?rshal;=20liblloyal=20=E2=86=92=20cee612d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last hop of the rc/partial contract, asserted on real weights from the JS side. test/integration.ts gains testDecodeFailure(): a cohort whose third scatter chunk finds no KV slot rejects with rc 1 and partial true — the landed children moved and were charged, the refused one did not and still prefills, prune returns the pool to zero; a 300-token prefill that dies after four 64-token chunks reports partial with its books unmoved, and a fresh branch prefills after prune. Both run on the default model and, when present, on Qwen3.5-4B (the production default, a Gated DeltaNet hybrid). The media rail is asserted per entry: an image costing 576 cells against a 256-cell pool comes back with error, rc 1 and partial true, and prune reclaims the branch. nSeqMax stays at 4 or 2. _storePrefillMultimodal's duplicate-handle guard compared DoubleValue() while the marshal coerced with Uint32Value(), so 5 and 5.5 passed the guard and named one branch. The guard coerces the same way now; the integration test asserts the fractional duplicate is rejected. tsconfig.test.json emitted JavaScript beside the sources on every build:test — outDir "." pulled the two src/ modules the unit test imports into the emit. The gate is noEmit: nothing consumed the output (the suites run through tsx). loadBinary's `variant` doc matches GpuVariant. Submodule pointer: cee612d (create refuses a no-vocab model; rc -1 restores too; the real-weights failure suite). Addon rebuilt against it. --- liblloyal | 2 +- src/SessionContext.cpp | 9 ++- src/index.ts | 3 +- test/integration.ts | 139 +++++++++++++++++++++++++++++++++++++++++ tsconfig.test.json | 8 +-- 5 files changed, 150 insertions(+), 11 deletions(-) diff --git a/liblloyal b/liblloyal index dd4667e..cee612d 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit dd4667e1f67008386ffdfdd5db163d2e1e8feb25 +Subproject commit cee612de0aab7bf5a3f854eb19c764fbce357d12 diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index 651a515..cb6bd53 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -726,7 +726,8 @@ class StorePrefillMultimodalWorker : public Napi::AsyncWorker { * before its decode ran (bitmap decode, marker/bitmap mismatch: the * branch is untouched) or inside decode_segments. `rc` and `partial` * say which case a decode failure is (see DecodeError in liblloyal): - * intact iff rc == 1 && !partial; anything else ⇒ prune and replay from + * intact iff the failing call restored state (rc 1 or -1) and nothing + * before it landed (!partial); anything else ⇒ prune and replay from * content — pruning an untouched branch is safe. */ std::string error; /** llama_decode's raw return code when the failure carried one (0 @@ -2776,10 +2777,12 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in // dispatches one handle at a time so the pair never meets there — check it // here to keep the same fail-loud contract as _storePrefill/_storeCommit. { - std::vector seen; + // Coerced exactly as the marshal below coerces them: 5 and 5.5 name the + // same branch, and the guard must see that. + std::vector seen; seen.reserve(n); for (uint32_t i = 0; i < n; i++) { - const double h = jsHandles.Get(i).As().DoubleValue(); + const uint32_t h = jsHandles.Get(i).As().Uint32Value(); for (uint32_t j = 0; j < seen.size(); j++) { if (seen[j] == h) { throw Napi::Error::New(env, diff --git a/src/index.ts b/src/index.ts index e11a74b..cc07c0d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -194,7 +194,8 @@ const claimImage = (binding: NativeBinding): NativeBinding => { * failure throws and never falls through — a corrupt pack must not * silently degrade to the npm CPU package. * - * @param variant GPU variant: 'cuda', 'vulkan', or undefined for CPU + * @param variant GPU variant — `'cuda'`, `'vulkan'` or `'default'`; `undefined` + * resolves from `LLOYAL_GPU` and the platform (see {@link GpuVariant}) * @returns Native binary module with createContext method * @throws Error if no binary is available for the current platform, or if a * DIFFERENT addon image is already loaded in this thread — see {@link diff --git a/test/integration.ts b/test/integration.ts index 1ef8147..3731462 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -2447,6 +2447,15 @@ async function testMultimodal(): Promise { } catch { dupThrew = true; } assert(dupThrew, 'duplicate handle in one _storePrefill rejects'); + // The multimodal cohort coerces handles with Uint32Value: 5 and 5.5 name + // the SAME branch, and its duplicate guard must see that too. + let fracDupThrew = false; + try { + await mm._storePrefillMultimodal( + [root.handle, root.handle + 0.5], [[], []], [userPrompt, userPrompt], [[IMG], [IMG]]); + } catch { fracDupThrew = true; } + assert(fracDupThrew, 'fractional duplicate handle in one _storePrefillMultimodal rejects'); + // Marker/bitmap count mismatch is reported PER ENTRY, not by rejecting the // whole call. A cohort carries N independent branches: rejecting would lose // which of them landed, and six agents settling images must not lose five @@ -2574,6 +2583,135 @@ async function testMultimodal(): Promise { } } +// ============================================================================ +// A failed decode says what landed — the JavaScript side of the kernel's +// DecodeError. The kernel's own suite proves the producers; this proves the +// last hop: `rc` and `partial` arrive on the rejection (token rail) and on +// the per-entry result (media rail), and the books a caller reads agree. +// Runs on the default model and, when present, on the production default +// (Qwen3.5-4B: a Gated DeltaNet hybrid, the carrier that cannot be rewound — +// the reason the contract is prune-and-replay in the first place). +// ============================================================================ +async function testDecodeFailure(): Promise { + console.log('\n--- Decode failure: partial ---'); + const failureOf = (err: unknown): { rc?: number; partial?: boolean; message?: string } => + err as { rc?: number; partial?: boolean; message?: string }; + const pressure = (c: SessionContext): { cellsUsed: number } => + (c as unknown as { _storeKvPressure(): { cellsUsed: number } })._storeKvPressure(); + const filler = (n: number, base: number, vocab: number): number[] => + Array.from({ length: n }, (_, j) => (base + j) % vocab); + + const models: string[] = [MODEL_PATH]; + const gdn = path.join(__dirname, '../models/Qwen3.5-4B-Q4_K_M.gguf'); + if (path.resolve(gdn) !== path.resolve(MODEL_PATH) && fs.existsSync(gdn)) models.push(gdn); + + for (const modelPath of models) { + console.log(` model: ${path.basename(modelPath)}`); + + // A later scatter chunk finds no KV slot: rc 1, partial true, the landed + // children moved, the refused one did not and still works. + { + const ctx: SessionContext = await addon.createContext({ modelPath, nCtx: 256, nBatch: 128, nSeqMax: 4, nThreads: 4 }); + try { + const vocab = ctx.vocabSize; + const prompt = await ctx.tokenize('Hello'); + assert(prompt.length > 0 && prompt.length < 40, 'scatter: a short prefix'); + const store = new BranchStore(ctx); + const root = Branch.create(ctx, 0, { temperature: 0 }); + await root.prefill(prompt); + const p = root.position; + const cells0 = pressure(ctx).cellsUsed; + const kids = [await root.fork(), await root.fork(), await root.fork()]; + // 100 tokens each under nBatch 128: three chunks. 256 cells hold p+200, not p+300. + let caught: ReturnType | null = null; + try { + await store.prefill([ + [kids[0], filler(100, 1000, vocab)], + [kids[1], filler(100, 2000, vocab)], + [kids[2], filler(100, 3000, vocab)], + ]); + } catch (err) { caught = failureOf(err); } + assert(caught !== null, 'scatter: the third chunk is refused'); + assert(caught!.rc === 1, `scatter: rc 1 on the rejection (got ${caught!.rc})`); + assert(caught!.partial === true, `scatter: partial:true on the rejection (got ${caught!.partial})`); + assert(kids[0].position === p + 100 && kids[1].position === p + 100, 'scatter: landed children moved'); + assert(kids[2].position === p, 'scatter: the refused child did not move'); + assert(pressure(ctx).cellsUsed === cells0 + 200, 'scatter: landed cells charged, refused cells not'); + await store.prefill([[kids[2], filler(10, 4000, vocab)]]); + assert(kids[2].position === p + 10, 'scatter: the intact child still prefills'); + await root.pruneSubtree(); + assert(pressure(ctx).cellsUsed === 0, 'scatter: the pool is whole after prune'); + ok('scatter: rc 1 + partial:true in JS; landed 2/3, intact child prefilled, pool reclaimed'); + } finally { + ctx.dispose(); + } + } + + // A chunked prefill that dies mid-way: poisoned, says so, prune reclaims it. + { + const ctx: SessionContext = await addon.createContext({ modelPath, nCtx: 256, nBatch: 64, nSeqMax: 2, nThreads: 4 }); + try { + const vocab = ctx.vocabSize; + const b = Branch.create(ctx, 0, { temperature: 0 }); + let caught: ReturnType | null = null; + try { await b.prefill(filler(300, 1000, vocab)); } catch (err) { caught = failureOf(err); } + assert(caught !== null && caught.rc === 1 && caught.partial === true, + `prefill: rc 1 + partial:true (got rc=${caught?.rc} partial=${caught?.partial})`); + assert(b.position === 0, 'prefill: the books did not move'); + assert(pressure(ctx).cellsUsed === 0, 'prefill: nothing charged'); + await b.prune(); + const fresh = Branch.create(ctx, 0, { temperature: 0 }); + await fresh.prefill(filler(10, 5000, vocab)); + assert(fresh.position === 10 && pressure(ctx).cellsUsed === 10, 'prefill: prune reclaimed the orphans — a fresh branch prefills'); + await fresh.prune(); + ok('prefill: poisoned reported as partial:true, reclaimed by prune'); + } finally { + ctx.dispose(); + } + } + } + + // The media rail reports PER ENTRY: an image whose cells outrun the pool + // comes back with error + rc 1 + partial:true, its siblings untouched. + const vl = pickVlPair(); + const bigImage = path.join(__dirname, '../liblloyal/tests/fixtures/cat.jpg'); + if (!vl || !fs.existsSync(bigImage)) { + console.log(' [SKIP] media: no VL pair or no large fixture'); + return; + } + { + const ctx: SessionContext = await addon.createContext({ + modelPath: vl.model, mmprojPath: vl.mmproj, nCtx: 256, nBatch: 16, nSeqMax: 2, nThreads: 4, + } as never); + try { + const mm = ctx as unknown as { + _storePrefillMultimodal(handles: number[], seps: number[][], prompts: string[], bitmaps: Buffer[][]): + Promise>; + _cellsMultimodal(sep: number[], prompt: string, bitmaps: Buffer[]): Promise; + }; + const IMG: Buffer = fs.readFileSync(bigImage); + const { prompt } = await ctx.formatChat(JSON.stringify([ + { role: 'user', content: [{ type: 'text', text: 'Describe this image.' }, { type: 'media_marker', text: MEDIA_MARKER }] }, + ])); + const cells = await mm._cellsMultimodal([], prompt, [IMG]); + if (cells <= 256) { + console.log(` [SKIP] media: the image costs ${cells} cells and fits in 256`); + } else { + const b = Branch.create(ctx, 0, { temperature: 0 }); + const [r] = await mm._storePrefillMultimodal([b.handle], [[]], [prompt], [[IMG]]); + assert(typeof r.error === 'string', 'media: the entry failed'); + assert(r.rc === 1 && r.partial === true, + `media: entry carries rc 1 + partial:true (got rc=${r.rc} partial=${r.partial}: ${r.error})`); + await b.prune(); + assert(pressure(ctx).cellsUsed === 0, 'media: prune reclaimed the poisoned branch'); + ok(`media: ${cells} cells against a 256-cell pool → per-entry rc 1 + partial:true, reclaimed`); + } + } finally { + ctx.dispose(); + } + } +} + async function main(): Promise { let mainCtx: SessionContext | null = null; @@ -2616,6 +2754,7 @@ async function main(): Promise { await testSetGrammar(); await testBranchMetrics(); await testMultimodal(); + await testDecodeFailure(); await testRerank(); await testRerankLargeCorpus(); await testRerankConcurrent(); diff --git a/tsconfig.test.json b/tsconfig.test.json index 53026e7..e54e48a 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -2,12 +2,8 @@ "extends": "./tsconfig.json", "compilerOptions": { "rootDir": ".", - "outDir": ".", - "declaration": false, - "declarationMap": false, - "sourceMap": false, - "skipLibCheck": true, - "noEmitOnError": false + "noEmit": true, + "skipLibCheck": true }, "include": ["test/**/*.ts"], "exclude": ["node_modules", "dist", "build", "test/sdk-agents.ts", "test/sdk-primitives.ts"] From fddd5cb2772b7333e4344b4e834cce44058c8a92 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 02:23:41 +1000 Subject: [PATCH 42/54] =?UTF-8?q?test:=20the=20compile=20gate=20emits=20ag?= =?UTF-8?q?ain=20=E2=80=94=20the=20GPU=20job=20runs=20test/integration.js?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 6962f5c made tsconfig.test.json noEmit on the belief that nothing consumed the emitted JS. The GPU deploy does: it runs /app/test/integration.js and /app/test/examples.js in the container, and the L4 job failed with 'Cannot find module'. Emission is back exactly as it was. The reason it was turned off stays fixed another way: src/backend-pack.js and src/verify.js appeared beside their sources because test/backend-pack-unit.ts imported ../src/…, pulling those modules into the test program's emit. It imports ../dist/… now, as integration.ts already does, so the source tree stays clean and the unit test runs over the built output it ships. --- test/backend-pack-unit.ts | 4 ++-- tsconfig.test.json | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/backend-pack-unit.ts b/test/backend-pack-unit.ts index 48423f2..aac0142 100644 --- a/test/backend-pack-unit.ts +++ b/test/backend-pack-unit.ts @@ -18,7 +18,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import * as zlib from 'node:zlib'; -import { sha256Hex, verifyPlatformSignature } from '../src/verify'; +import { sha256Hex, verifyPlatformSignature } from '../dist/verify.js'; import { backendPackCacheDir, detectCudaRuntime, @@ -29,7 +29,7 @@ import { resolveBackendPackDirSync, type BackendPackManifest, type CommandRunner, -} from '../src/backend-pack'; +} from '../dist/backend-pack.js'; let passed = 0; async function test(name: string, fn: () => void | Promise): Promise { diff --git a/tsconfig.test.json b/tsconfig.test.json index e54e48a..53026e7 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -2,8 +2,12 @@ "extends": "./tsconfig.json", "compilerOptions": { "rootDir": ".", - "noEmit": true, - "skipLibCheck": true + "outDir": ".", + "declaration": false, + "declarationMap": false, + "sourceMap": false, + "skipLibCheck": true, + "noEmitOnError": false }, "include": ["test/**/*.ts"], "exclude": ["node_modules", "dist", "build", "test/sdk-agents.ts", "test/sdk-primitives.ts"] From 1e969322970f4b3c41cc9647cd878f124ba134c1 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 02:31:27 +1000 Subject: [PATCH 43/54] =?UTF-8?q?binding:=20the=20multimodal=20cohort=20ap?= =?UTF-8?q?plies=20the=20kernel's=20distinct-handles=20rule;=20liblloyal?= =?UTF-8?q?=20=E2=86=92=20261200d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The duplicate guard was a nested scan over a second coercion of the handles — the binding's own statement of a rule the kernel owns, made only because this path dispatches one branch at a time and decode_scatter never sees the pair. It is gone. The handles are marshaled first, once, and lloyal::branch::require_distinct_handles runs over exactly the values that will be dispatched — the same one-pass rule decode_each and decode_scatter now apply. _storeCommit needs nothing here: the kernel's decode_each refuses a repeated handle itself now, and the worker's catch forwards it. Submodule pointer: 261200d. Addon rebuilt; the fractional-duplicate and partial assertions pass on real weights. --- liblloyal | 2 +- src/SessionContext.cpp | 32 ++++++++++---------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/liblloyal b/liblloyal index cee612d..261200d 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit cee612de0aab7bf5a3f854eb19c764fbce357d12 +Subproject commit 261200d06940b87b086525a302a1a42619ff836a diff --git a/src/SessionContext.cpp b/src/SessionContext.cpp index cb6bd53..5e823be 100644 --- a/src/SessionContext.cpp +++ b/src/SessionContext.cpp @@ -2772,28 +2772,6 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in return deferred.Promise(); } - // Duplicate handles would prefill the same branch twice in sequence, each - // advancing its position. decode_scatter rejects duplicates, but this path - // dispatches one handle at a time so the pair never meets there — check it - // here to keep the same fail-loud contract as _storePrefill/_storeCommit. - { - // Coerced exactly as the marshal below coerces them: 5 and 5.5 name the - // same branch, and the guard must see that. - std::vector seen; - seen.reserve(n); - for (uint32_t i = 0; i < n; i++) { - const uint32_t h = jsHandles.Get(i).As().Uint32Value(); - for (uint32_t j = 0; j < seen.size(); j++) { - if (seen[j] == h) { - throw Napi::Error::New(env, - "_storePrefillMultimodal: duplicate handle at indices " + - std::to_string(j) + " and " + std::to_string(i)); - } - } - seen.push_back(h); - } - } - // Marshal everything on the JS thread — the worker owns copies (Buffers // must never be touched off-thread). std::vector handles(n); @@ -2838,6 +2816,16 @@ Napi::Value SessionContext::_storePrefillMultimodal(const Napi::CallbackInfo& in } } + // A handle may appear at most once per call — the kernel's rule, applied + // to the marshaled handles: this path dispatches one branch at a time, so + // decode_scatter never sees the pair, and the values checked are exactly + // the values dispatched. + try { + lloyal::branch::require_distinct_handles(handles, "_storePrefillMultimodal"); + } catch (const std::exception& e) { + throw Napi::Error::New(env, e.what()); + } + const int32_t nEmbdInp = llama_model_n_embd_inp(_model.get()); auto* worker = new StorePrefillMultimodalWorker( From d61313a31924b18b729d985c836a2f16ff8749dc Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 02:40:18 +1000 Subject: [PATCH 44/54] =?UTF-8?q?chore:=20liblloyal=20=E2=86=92=2018c661e?= =?UTF-8?q?=20(the=20repeated-handle=20rule=20witnessed=20on=20real=20weig?= =?UTF-8?q?hts)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 261200d..18c661e 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 261200d06940b87b086525a302a1a42619ff836a +Subproject commit 18c661e2290410285d38eb2e75d084cc049781a5 From 740c8d8909ca0f0c37d69799203157723d18080e Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 02:53:57 +1000 Subject: [PATCH 45/54] =?UTF-8?q?chore:=20liblloyal=20=E2=86=92=20f1f8b01?= =?UTF-8?q?=20(a=20refused=20create=20owns=20nothing;=20n=5Fubatch=20refus?= =?UTF-8?q?al=20tested;=20CI=20runs=20the=20embedding-rail=20failure=20cas?= =?UTF-8?q?e)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index 18c661e..f1f8b01 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit 18c661e2290410285d38eb2e75d084cc049781a5 +Subproject commit f1f8b01f07f4edcf31c8973fba9032f6ecfa1ec8 From f17ac419ccca363fefd4a7c3df80282f586a026a Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 03:17:12 +1000 Subject: [PATCH 46/54] =?UTF-8?q?chore:=20liblloyal=20=E2=86=92=20d525b22?= =?UTF-8?q?=20(the=20context's=20batch=20is=20refused=20before=20it=20is?= =?UTF-8?q?=20built;=20embedding-rail=20case=20fits=20every=20projector)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- liblloyal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblloyal b/liblloyal index f1f8b01..d525b22 160000 --- a/liblloyal +++ b/liblloyal @@ -1 +1 @@ -Subproject commit f1f8b01f07f4edcf31c8973fba9032f6ecfa1ec8 +Subproject commit d525b224741fcaf438a7db43198ec8be33ff2612 From 3224ca8b90a67bd9d6d773a2e54c61f59bba8634 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 03:27:07 +1000 Subject: [PATCH 47/54] =?UTF-8?q?ci:=20build=20TypeScript=20before=20the?= =?UTF-8?q?=20unit=20tests=20=E2=80=94=20they=20run=20against=20dist,=20as?= =?UTF-8?q?=20integration.ts=20always=20has?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests.yml's matrix job ran test:unit straight after npm install --ignore-scripts, before any build. That only ever worked because tsx compiled the test's ../src imports in memory; 6962f5c moved those imports to ../dist so the compile gate stops emitting .js beside the sources, and this job had no dist to import. Every other workflow already builds before testing. --- .github/workflows/tests.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a1881f4..49568ae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,7 +67,14 @@ jobs: # cache resolution, and the pack extractor. Plus the arch-mirror # drift gate: fails when a llama.cpp sync changes ggml's CUDA arch # derivation without scripts/dl-archs.js being reconciled. - - name: Backend-pack unit tests + arch drift gate + stderr allowlist + # The test entries run against dist, the output that ships — integration.ts + # always did, and the unit test does since 6962f5c (importing ../src from a + # test made the compile gate emit .js beside the sources). tsc only; no + # native build is needed for this. + - name: Build TypeScript + run: npm run build:ts + + - name: Backend-pack unit tests + arch drift gate + stderr allowlist if: runner.os == 'Linux' run: | npm run test:unit From f321515d3374334a604c0157e29b91a8a3c13ca1 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 03:28:40 +1000 Subject: [PATCH 48/54] ci: fix the indentation of the Build TypeScript step (3224ca8 was not valid YAML) --- .github/workflows/tests.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 49568ae..f6b7dbf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,14 +67,15 @@ jobs: # cache resolution, and the pack extractor. Plus the arch-mirror # drift gate: fails when a llama.cpp sync changes ggml's CUDA arch # derivation without scripts/dl-archs.js being reconciled. - # The test entries run against dist, the output that ships — integration.ts - # always did, and the unit test does since 6962f5c (importing ../src from a - # test made the compile gate emit .js beside the sources). tsc only; no - # native build is needed for this. - - name: Build TypeScript - run: npm run build:ts - - - name: Backend-pack unit tests + arch drift gate + stderr allowlist + # + # The test entries run against dist, the output that ships — + # integration.ts always did, and the unit test does since 6962f5c + # (importing ../src from a test made the compile gate emit .js beside + # the sources). tsc only; no native build is needed for this. + - name: Build TypeScript + run: npm run build:ts + + - name: Backend-pack unit tests + arch drift gate + stderr allowlist if: runner.os == 'Linux' run: | npm run test:unit From 64195a07d030ec09f1e8759d7b0fbfc08e67f352 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 12:57:40 +1000 Subject: [PATCH 49/54] fix(node,docs): correct the store loop, re-export list, fan-out assertion, cover tokenToBytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BranchStore.commit already accepts every token before decoding, so the README's Branch.accept() advanced grammar, sampler and metrics twice. Drop it and let the commit do the accept once, matching the SDK's own store loop. The re-export list named formatChat, parseChatOutput, jsonSchemaToGrammar and per-token metrics as top-level values; those are methods on the context and branch, not exports. List what src/index.ts actually re-exports so a consumer following it does not hit missing exports. The fan-out test asserted ticks > 1, which a lone survivor decoding serially would satisfy without the sustained four-way batching under test. Record each dispatch width and require more than one dispatch that carried several branches. tokenToBytes, new on this branch, had no integration coverage. Assert a Buffer per token, that at least one piece ends mid-character, and that the bytes concatenate back to detokenize() — the byte API's contract, which tokenToText cannot hold for split multibyte characters. --- README.md | 6 +++--- test/integration.ts | 31 ++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 802bbd2..5da85b8 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ for (;;) { const items = produced .filter((p) => !p.isStop) - .map((p) => { p.b.accept(p.token); return [p.b, p.token]; }); - if (items.length) await store.commit(items); // N branches, 1 llama_decode() + .map((p) => [p.b, p.token]); + if (items.length) await store.commit(items); // accept + decode: N branches, 1 llama_decode() } ``` @@ -82,7 +82,7 @@ lloyal.node binds [liblloyal](https://github.com/lloyal-ai/liblloyal) — the C+ **What it re-exports**, so one install is enough — these are [HDK](https://github.com/lloyal-ai/hdk) packages, documented there: -- from `@lloyal-labs/sdk`: `Branch`, `BranchStore`, `Session`, `Rerank`, `formatChat`, `parseChatOutput`, `jsonSchemaToGrammar`, per-token metrics +- from `@lloyal-labs/sdk`: `Branch`, `BranchStore`, `Session`, `Rerank`, `buildUserDelta`, `buildToolResultDelta`, and the sampling, chat and rerank types - from `@lloyal-labs/lloyal-agents`: `Tool`, `Agent`, `agent`, `agentPool`, `useAgent`, `useAgentPool`, `withSpine`, `diverge`, `reduce`, `createToolkit`, `initAgents`, `DefaultAgentPolicy`, `renderTemplate` Not re-exported — import from the package itself: the Ability protocol (`AbilityRegistryCtx`, `AbilityConfigStoreCtx`, `AbilityManifest`, `GrantStoreCtx`) from `@lloyal-labs/lloyal-agents`, and `defineAbility` / `createAbilityRegistry` / `createGrantStore` from `@lloyal-labs/rig`. diff --git a/test/integration.ts b/test/integration.ts index 3731462..f5146ba 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -740,6 +740,22 @@ async function testTokenizer(ctx: SessionContext): Promise { const eogText: string = ctx.tokenToText(eog); assert(eogText.length > 0, `EOS text: "${eogText}"`); + // tokenToBytes — the byte-level twin of tokenToText. A BPE piece can end + // mid-character: decoded one at a time the pieces tear into U+FFFD, while + // their bytes concatenate back to the text. (SentencePiece models keep the + // prefix space on the first piece that detokenize() strips.) + const multibyte = '日本語 🎉 𠜎𠜱 👨‍👩‍👧‍👦'; + const mbTokens: number[] = await ctx.tokenize(multibyte, false); + const pieces: Uint8Array[] = mbTokens.map((t: number) => ctx.tokenToBytes(t)); + assert(pieces.every((p: Uint8Array) => Buffer.isBuffer(p)), + `tokenToBytes returns a Buffer per token (${pieces.length} pieces)`); + const torn: number = pieces.filter((p: Uint8Array) => Buffer.from(p).toString('utf8').includes('\uFFFD')).length; + assert(torn > 0, `${torn} of ${pieces.length} pieces end mid-character`); + const joined: string = Buffer.concat(pieces).toString('utf8'); + const detok: string = await ctx.detokenize(mbTokens); + assert(joined === detok || joined === ' ' + detok, + `bytes concatenate to detokenize(): ${JSON.stringify(joined)}`); + // tokenize with addSpecial const withSpecial: number[] = await ctx.tokenize('Hello world', true); const noSpecial: number[] = await ctx.tokenize('Hello world', false); @@ -2541,7 +2557,7 @@ async function testMultimodal(): Promise { // One store.commit() per tick carries every live child. const fanToks: number[][] = asks.map(() => []); - let ticks = 0; + const widths: number[] = []; // branches carried by each dispatch for (let step = 0; step < 16; step++) { const entries: Array<[InstanceType, number]> = []; for (let i = 0; i < kids.length; i++) { @@ -2553,13 +2569,14 @@ async function testMultimodal(): Promise { } if (!entries.length) break; await store.commit(entries); - ticks++; + widths.push(entries.length); } - // >1 is the point: several consecutive ticks each carrying every live - // branch in ONE llama_decode is continuous tree batching. A single - // dispatch would satisfy "it batched" without showing it sustains. - assert(ticks > 1, - `fan-out: ${ticks} batched dispatches, each carrying up to ${kids.length} branches`); + // Sustained batching is several dispatches that each carry SEVERAL + // branches. Counting non-empty ticks would pass when three children stop + // at tick one and a lone survivor runs on — serial decoding, not batching. + const batched: number = widths.filter((w) => w > 1).length; + assert(batched > 1, + `fan-out: dispatch widths ${widths.join(',')} — ${batched} carried several branches`); for (let i = 0; i < asks.length; i++) { const toks: number[] = fanToks[i].filter((t) => t !== -1); From f9ef1eb74936f6957a3b5e603bef2e2277e4938b Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 19:43:26 +1000 Subject: [PATCH 50/54] test: the GPU job compiles against the published sdk types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test/integration.ts` called `ctx.tokenToBytes`, which the linked hdk workspace's SessionContext declares but the `^3` devDependency CI installs (sdk 3.1.0) does not — TS2339 on the L4 job while `npm run build:test` was green locally, because the local package is a symlink to the workspace. The call is typed structurally, the convention this file already uses for `_storePrefill*`: binding surface added ahead of the sdk pin is named at the test site until the pin catches up. Reproduced CI's exact error against published 3.1.0 before the change; green under both types afterwards. --- test/integration.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration.ts b/test/integration.ts index f5146ba..891d208 100644 --- a/test/integration.ts +++ b/test/integration.ts @@ -744,9 +744,13 @@ async function testTokenizer(ctx: SessionContext): Promise { // mid-character: decoded one at a time the pieces tear into U+FFFD, while // their bytes concatenate back to the text. (SentencePiece models keep the // prefix space on the first piece that detokenize() strips.) + // Typed structurally, as the multimodal block does for `_storePrefill*`: + // the published sdk's SessionContext (the `^3` devDependency CI installs) + // predates this method, and the linked workspace hides that locally. + const bytes = ctx as unknown as { tokenToBytes(token: number): Uint8Array }; const multibyte = '日本語 🎉 𠜎𠜱 👨‍👩‍👧‍👦'; const mbTokens: number[] = await ctx.tokenize(multibyte, false); - const pieces: Uint8Array[] = mbTokens.map((t: number) => ctx.tokenToBytes(t)); + const pieces: Uint8Array[] = mbTokens.map((t: number) => bytes.tokenToBytes(t)); assert(pieces.every((p: Uint8Array) => Buffer.isBuffer(p)), `tokenToBytes returns a Buffer per token (${pieces.length} pieces)`); const torn: number = pieces.filter((p: Uint8Array) => Buffer.from(p).toString('utf8').includes('\uFFFD')).length; From beec2f99317c9a4ce04927c4947ea57333d93c00 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 20:26:16 +1000 Subject: [PATCH 51/54] =?UTF-8?q?alpha:=20cut=202=20=E2=80=94=203.2.0-alph?= =?UTF-8?q?a.2,=20platform=20packages=20follow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The set id moves from alpha.1 to alpha.2 for this arc's second cut. The main package and every optionalDependency on a platform package carry the same version, kept in step by scripts/sync-versions.js, so npm resolves one coherent set. `latest` is untouched: the Release workflow maps an -alpha version to the alpha dist-tag. --- package-lock.json | 4 ++-- package.json | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 350f9e3..1f5020a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.1", + "version": "3.2.0-alpha.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.1", + "version": "3.2.0-alpha.2", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@lloyal-labs/tsampler": "^0.2.0", diff --git a/package.json b/package.json index 2fbc6d9..9863538 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.1", + "version": "3.2.0-alpha.2", "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -67,19 +67,19 @@ "typescript": "^5.9.3" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.2" }, "engines": { "node": ">=24.0.0" From 680793e7edd4abe6c23b0f05be6dc733d5349b7c Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 22:49:09 +1000 Subject: [PATCH 52/54] =?UTF-8?q?alpha:=20cut=203=20=E2=80=94=20the=20bind?= =?UTF-8?q?ing's=20peers=20admit=20the=20set=20it=20ships=20beside;=203.2.?= =?UTF-8?q?0-alpha.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dist/index.js re-exports values from @lloyal-labs/sdk and @lloyal-labs/lloyal-agents, so both are real peers. They were declared as `>=3.0.0`, and a plain range never admits a prerelease: semver matches one only when a comparator names that exact major.minor.patch with a prerelease tag. Every `npm install` that pinned the alpha set beside this binding — the scaffold's own install included — was refused (ERESOLVE), root-declared or not. The ranges now name the tuples: `>=3.0.0 || >=4.0.0-0 <5.0.0` on sdk and `>=3.0.0 || >=6.0.0-0 <7.0.0` on agents. Each admits the stable a user has today, this arc's prerelease, and the stable it becomes. A unit test holds that, red first, checked with the semver library npm resolves with; it runs under `npm run test:unit` on every pull request. The version moves to 3.2.0-alpha.3 with the platform packages in step. Follow-up for the next release, not this cut: nothing imports those re-exports (the templates and hdk import only createContext and loadBinary), so removing them removes the load-time requires, the peers, and the bare-install landmine in one move. --- package-lock.json | 193 +++++++++++++++++++++++++++++++++++++-------- package.json | 35 ++++---- test/peers-unit.ts | 35 ++++++++ 3 files changed, 215 insertions(+), 48 deletions(-) create mode 100644 test/peers-unit.ts diff --git a/package-lock.json b/package-lock.json index 1f5020a..0bca560 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.2", + "version": "3.2.0-alpha.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.2", + "version": "3.2.0-alpha.3", "license": "SEE LICENSE IN LICENSE", "dependencies": { "@lloyal-labs/tsampler": "^0.2.0", @@ -18,6 +18,7 @@ "@types/node": "^25.3.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", + "semver": "^7.8.5", "tsx": "^4.21.0", "typedoc": "^0.28.16", "typedoc-rhineai-theme": "^1.2.0", @@ -27,19 +28,19 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.1", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.1" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.2", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.2" }, "peerDependencies": { "@lloyal-labs/lloyal-agents": ">=3.0.0", @@ -538,43 +539,173 @@ } }, "node_modules/@lloyal-labs/lloyal.node-darwin-arm64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-darwin-arm64/-/lloyal.node-darwin-arm64-3.2.0-alpha.2.tgz", + "integrity": "sha512-vv+LfEIL/TS0fjzkW/TUC/mjrlPa3XBQn9pmV4r02YFTpt4+9PT/VatJujx8hwo7WOmV6TeenPc/YlWr6ZJUxw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] }, "node_modules/@lloyal-labs/lloyal.node-darwin-x64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-darwin-x64/-/lloyal.node-darwin-x64-3.2.0-alpha.2.tgz", + "integrity": "sha512-voSS4Jn+lhChr+qFMPdYRs0MjE6eDP88XrUlp4P4Goch8R3ZFrKFzUXtws4Inhu4qWHUnngqweiegz+3uPXVYg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64/-/lloyal.node-linux-arm64-3.2.0-alpha.2.tgz", + "integrity": "sha512-wOjIbRAqNaIkUxAL0Al4jK8TJHa1jlgH4IsQepBLPdbUTW7zdib2nNW+zZWnK+zetBPiLUsBly4oVVgVxLGIPg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64-cuda": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64-cuda/-/lloyal.node-linux-arm64-cuda-3.2.0-alpha.2.tgz", + "integrity": "sha512-VIPdTwUnQp6+vJtaqhq42DetZJMqoAuud+RY8UoJvWGSK96TsAZlp4RyAc5E87QDFLHhkwfSVfNvZgn5q5FxJA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64-vulkan": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64-vulkan/-/lloyal.node-linux-arm64-vulkan-3.2.0-alpha.2.tgz", + "integrity": "sha512-9elCpjWYOZsxFjf+PpIZ8TyG0H40RfT1g4Nyq5UBJiJd7YB3OVnm7Ac+dooUVAUKXfWv360Zl1yc5j6Le94D6w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-x64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64/-/lloyal.node-linux-x64-3.2.0-alpha.2.tgz", + "integrity": "sha512-3kkhW9kUF/3jPBZdYvFas0vcA0xl50bAk9r+lMWfI1wGdNYN8V5U4dDD+e53MOjah/KwJbvL4b5GiTwp6gl+rA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-x64-cuda": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64-cuda/-/lloyal.node-linux-x64-cuda-3.2.0-alpha.2.tgz", + "integrity": "sha512-oevwYBQgtTcQFOqWFpDRvhq+riccAcD4EDHehkGchp3/c9at/OloFrRzBDoK2KC3CxUMXupkQu7eAA0bfnJfiQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-linux-x64-vulkan": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64-vulkan/-/lloyal.node-linux-x64-vulkan-3.2.0-alpha.2.tgz", + "integrity": "sha512-GrXSBmZle4vX4bioIoyQHGU7z7UCUnsmfks/wt+5l8Ll8ogcubOxjtFqyhFvxoM1NICgV0Z2irOsBxqA+WJppQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] }, "node_modules/@lloyal-labs/lloyal.node-win32-arm64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-arm64/-/lloyal.node-win32-arm64-3.2.0-alpha.2.tgz", + "integrity": "sha512-BwpG4w9KnPb2/lOKQ8T22QQBQiuKJzfXaKF08bqVG2MnPYAzp/Sa9lhmKHZ3SMbpB2FiXal+SwdIzowWSRRjnQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@lloyal-labs/lloyal.node-win32-arm64-vulkan": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-arm64-vulkan/-/lloyal.node-win32-arm64-vulkan-3.2.0-alpha.2.tgz", + "integrity": "sha512-IQsGuB4cg1hssTw42Tyt1Wx4ghcEk17j3QZWV+PDKqj4RCcvDsZc1XTiCbEFFfrfb6noSk76xru+cvRxKPL2xQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@lloyal-labs/lloyal.node-win32-x64": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64/-/lloyal.node-win32-x64-3.2.0-alpha.2.tgz", + "integrity": "sha512-kFyT5BkJLqGpvQPwUF0rIgEvfs4f2DADljenzZ6ez7usRBNf8tLJ+2VCcbxYjKOa7CaU8bsbwMmCjiEGEz37Mg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@lloyal-labs/lloyal.node-win32-x64-cuda": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64-cuda/-/lloyal.node-win32-x64-cuda-3.2.0-alpha.2.tgz", + "integrity": "sha512-tZ6QyYzX6A/K2W3UBT8oWvGnX0njWVDxDVgGLP0XNhS2ybZjRTxkZgbnuSURgHAqChpZUb6jOWdO8OD4E+Nf4w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@lloyal-labs/lloyal.node-win32-x64-vulkan": { - "optional": true + "version": "3.2.0-alpha.2", + "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64-vulkan/-/lloyal.node-win32-x64-vulkan-3.2.0-alpha.2.tgz", + "integrity": "sha512-wM7OTXLziVsyQ+CArwHm4CPg4IgozBHR0GGorEvkJWV7Zzo4+qqTQmE1N12w5gTGmF6dRGEfkZXxoU7cXPB5fQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@lloyal-labs/sdk": { "version": "3.0.0", @@ -1314,9 +1445,9 @@ } }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { diff --git a/package.json b/package.json index 9863538..634e2bd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lloyal-labs/lloyal.node", - "version": "3.2.0-alpha.2", + "version": "3.2.0-alpha.3", "description": "The Node runtime for the HDK — built on liblloyal and llama.cpp", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -20,7 +20,7 @@ "version": "node scripts/sync-versions.js && git add -A", "docs": "npx typedoc", "test": "npm run test:integration", - "test:unit": "npx tsx test/backend-pack-unit.ts", + "test:unit": "npx tsx test/backend-pack-unit.ts && npx tsx test/peers-unit.ts", "test:canary": "npx tsx test/stderr-canary.ts", "test:integration": "npx tsx test/integration.ts", "test:examples": "npx tsx test/examples.ts", @@ -52,8 +52,8 @@ "node-addon-api": "^8.5.0" }, "peerDependencies": { - "@lloyal-labs/lloyal-agents": ">=3.0.0", - "@lloyal-labs/sdk": ">=3.0.0" + "@lloyal-labs/lloyal-agents": ">=3.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/sdk": ">=3.0.0 || >=4.0.0-0 <5.0.0" }, "devDependencies": { "@lloyal-labs/lloyal-agents": "^3.0.0", @@ -61,25 +61,26 @@ "@types/node": "^25.3.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", + "semver": "^7.8.5", "tsx": "^4.21.0", "typedoc": "^0.28.16", "typedoc-rhineai-theme": "^1.2.0", "typescript": "^5.9.3" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.2" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.3" }, "engines": { "node": ">=24.0.0" diff --git a/test/peers-unit.ts b/test/peers-unit.ts new file mode 100644 index 0000000..84a7fd3 --- /dev/null +++ b/test/peers-unit.ts @@ -0,0 +1,35 @@ +/** + * The binding's peers must admit the set it ships beside — no GPU, no network. + * tsx + node:assert (the repo's test convention; run via `npm run test:unit`). + * + * dist/index.js re-exports values from @lloyal-labs/sdk and @lloyal-labs/lloyal-agents, + * so both are real peers. A plain range (`>=3.0.0`) never admits a prerelease: + * semver matches a prerelease only when a comparator names that exact + * major.minor.patch with a prerelease tag. So the alpha set (sdk 4.0.0-alpha.N, + * agents 6.0.0-alpha.N) failed every `npm install` that pinned it beside this + * binding — the scaffold's own install included — until the ranges named the + * tuples. Checked with the semver library npm resolves with. + */ +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { satisfies } from 'semver'; + +const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { + peerDependencies: Record; +}; +const peers = pkg.peerDependencies; + +/** Each peer: the stable a user has today, the prerelease this arc ships, the stable it becomes. */ +const ADMITS: Record = { + '@lloyal-labs/sdk': ['3.1.0', '4.0.0-alpha.3', '4.0.0'], + '@lloyal-labs/lloyal-agents': ['5.5.1', '6.0.0-alpha.3', '6.0.0'], +}; +for (const [name, versions] of Object.entries(ADMITS)) { + assert.ok(peers[name], `${name} is declared as a peer`); + for (const v of versions) { + assert.ok(satisfies(v, peers[name]), `${name} "${peers[name]}" admits ${v}`); + } +} +// The trap, stated: a plain range excludes the prerelease this set is made of. +assert.equal(satisfies('6.0.0-alpha.3', '>=3.0.0'), false); +console.log('peers-unit: ok'); From b1d62dcbaf1140e4248352fdbe3cd9a568d54b90 Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 23:29:07 +1000 Subject: [PATCH 53/54] test: the peers check compiles under the tests project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tsconfig.test.json compiles the tests as CommonJS, so the new peers test used two things tsx accepts and tsc rejects: `import.meta.url` (illegal under CommonJS) and an untyped `semver` import. It now reads package.json through `__dirname`, and @types/semver is a devDependency. The compile gate — `npm run build:test`, the same step the GPU job runs — was run before this commit, which is what should have happened the first time. --- package-lock.json | 194 +++++++++------------------------------------ package.json | 1 + test/peers-unit.ts | 5 +- 3 files changed, 41 insertions(+), 159 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0bca560..9947a46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@lloyal-labs/lloyal-agents": "^3.0.0", "@lloyal-labs/sdk": "^3.0.0", "@types/node": "^25.3.0", + "@types/semver": "^7.8.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", "semver": "^7.8.5", @@ -28,23 +29,23 @@ "node": ">=24.0.0" }, "optionalDependencies": { - "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.2", - "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.2" + "@lloyal-labs/lloyal.node-darwin-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-darwin-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-linux-x64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-arm64-vulkan": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-cuda": "3.2.0-alpha.3", + "@lloyal-labs/lloyal.node-win32-x64-vulkan": "3.2.0-alpha.3" }, "peerDependencies": { - "@lloyal-labs/lloyal-agents": ">=3.0.0", - "@lloyal-labs/sdk": ">=3.0.0" + "@lloyal-labs/lloyal-agents": ">=3.0.0 || >=6.0.0-0 <7.0.0", + "@lloyal-labs/sdk": ">=3.0.0 || >=4.0.0-0 <5.0.0" } }, "node_modules/@esbuild/aix-ppc64": { @@ -539,173 +540,43 @@ } }, "node_modules/@lloyal-labs/lloyal.node-darwin-arm64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-darwin-arm64/-/lloyal.node-darwin-arm64-3.2.0-alpha.2.tgz", - "integrity": "sha512-vv+LfEIL/TS0fjzkW/TUC/mjrlPa3XBQn9pmV4r02YFTpt4+9PT/VatJujx8hwo7WOmV6TeenPc/YlWr6ZJUxw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-darwin-x64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-darwin-x64/-/lloyal.node-darwin-x64-3.2.0-alpha.2.tgz", - "integrity": "sha512-voSS4Jn+lhChr+qFMPdYRs0MjE6eDP88XrUlp4P4Goch8R3ZFrKFzUXtws4Inhu4qWHUnngqweiegz+3uPXVYg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64/-/lloyal.node-linux-arm64-3.2.0-alpha.2.tgz", - "integrity": "sha512-wOjIbRAqNaIkUxAL0Al4jK8TJHa1jlgH4IsQepBLPdbUTW7zdib2nNW+zZWnK+zetBPiLUsBly4oVVgVxLGIPg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64-cuda": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64-cuda/-/lloyal.node-linux-arm64-cuda-3.2.0-alpha.2.tgz", - "integrity": "sha512-VIPdTwUnQp6+vJtaqhq42DetZJMqoAuud+RY8UoJvWGSK96TsAZlp4RyAc5E87QDFLHhkwfSVfNvZgn5q5FxJA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-arm64-vulkan": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-arm64-vulkan/-/lloyal.node-linux-arm64-vulkan-3.2.0-alpha.2.tgz", - "integrity": "sha512-9elCpjWYOZsxFjf+PpIZ8TyG0H40RfT1g4Nyq5UBJiJd7YB3OVnm7Ac+dooUVAUKXfWv360Zl1yc5j6Le94D6w==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-x64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64/-/lloyal.node-linux-x64-3.2.0-alpha.2.tgz", - "integrity": "sha512-3kkhW9kUF/3jPBZdYvFas0vcA0xl50bAk9r+lMWfI1wGdNYN8V5U4dDD+e53MOjah/KwJbvL4b5GiTwp6gl+rA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-x64-cuda": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64-cuda/-/lloyal.node-linux-x64-cuda-3.2.0-alpha.2.tgz", - "integrity": "sha512-oevwYBQgtTcQFOqWFpDRvhq+riccAcD4EDHehkGchp3/c9at/OloFrRzBDoK2KC3CxUMXupkQu7eAA0bfnJfiQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-linux-x64-vulkan": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-linux-x64-vulkan/-/lloyal.node-linux-x64-vulkan-3.2.0-alpha.2.tgz", - "integrity": "sha512-GrXSBmZle4vX4bioIoyQHGU7z7UCUnsmfks/wt+5l8Ll8ogcubOxjtFqyhFvxoM1NICgV0Z2irOsBxqA+WJppQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-win32-arm64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-arm64/-/lloyal.node-win32-arm64-3.2.0-alpha.2.tgz", - "integrity": "sha512-BwpG4w9KnPb2/lOKQ8T22QQBQiuKJzfXaKF08bqVG2MnPYAzp/Sa9lhmKHZ3SMbpB2FiXal+SwdIzowWSRRjnQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-win32-arm64-vulkan": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-arm64-vulkan/-/lloyal.node-win32-arm64-vulkan-3.2.0-alpha.2.tgz", - "integrity": "sha512-IQsGuB4cg1hssTw42Tyt1Wx4ghcEk17j3QZWV+PDKqj4RCcvDsZc1XTiCbEFFfrfb6noSk76xru+cvRxKPL2xQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-win32-x64": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64/-/lloyal.node-win32-x64-3.2.0-alpha.2.tgz", - "integrity": "sha512-kFyT5BkJLqGpvQPwUF0rIgEvfs4f2DADljenzZ6ez7usRBNf8tLJ+2VCcbxYjKOa7CaU8bsbwMmCjiEGEz37Mg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-win32-x64-cuda": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64-cuda/-/lloyal.node-win32-x64-cuda-3.2.0-alpha.2.tgz", - "integrity": "sha512-tZ6QyYzX6A/K2W3UBT8oWvGnX0njWVDxDVgGLP0XNhS2ybZjRTxkZgbnuSURgHAqChpZUb6jOWdO8OD4E+Nf4w==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] + "optional": true }, "node_modules/@lloyal-labs/lloyal.node-win32-x64-vulkan": { - "version": "3.2.0-alpha.2", - "resolved": "https://registry.npmjs.org/@lloyal-labs/lloyal.node-win32-x64-vulkan/-/lloyal.node-win32-x64-vulkan-3.2.0-alpha.2.tgz", - "integrity": "sha512-wM7OTXLziVsyQ+CArwHm4CPg4IgozBHR0GGorEvkJWV7Zzo4+qqTQmE1N12w5gTGmF6dRGEfkZXxoU7cXPB5fQ==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ] + "optional": true }, "node_modules/@lloyal-labs/sdk": { "version": "3.0.0", @@ -789,6 +660,13 @@ "undici-types": "~7.19.0" } }, + "node_modules/@types/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", diff --git a/package.json b/package.json index 634e2bd..561dbbd 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "@lloyal-labs/lloyal-agents": "^3.0.0", "@lloyal-labs/sdk": "^3.0.0", "@types/node": "^25.3.0", + "@types/semver": "^7.8.0", "cmake-js": "^8.0.0", "glob": "^11.0.0", "semver": "^7.8.5", diff --git a/test/peers-unit.ts b/test/peers-unit.ts index 84a7fd3..7d47c66 100644 --- a/test/peers-unit.ts +++ b/test/peers-unit.ts @@ -12,9 +12,12 @@ */ import { strict as assert } from 'node:assert'; import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { satisfies } from 'semver'; -const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { +// CommonJS on purpose: tsconfig.test.json compiles the tests as CommonJS, so +// `__dirname`, not `import.meta.url` — the compile gate, not tsx, is the judge. +const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8')) as { peerDependencies: Record; }; const peers = pkg.peerDependencies; From 9b0403d17e6fb105a7ea7b9b0e011063ccfb022b Mon Sep 17 00:00:00 2001 From: lloyal-research Date: Fri, 4 Sep 2026 23:53:48 +1000 Subject: [PATCH 54/54] ci(release): the DL chain can be resumed for a version whose archives already uploaded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pack is deterministic and the channel's paths are write-once, so a DL chain that fails AFTER its upload — as 3.2.0-alpha.3's did, on a test compile error — could not be finished at that version: a second run's upload answers 409 and never reaches the L4 gate or the signing act. Until now the only remedy was the next version number. `dl_source_run_id` names the earlier run. A small job downloads that run's manifest artifact, refuses a version other than this checkout's or a manifest already published, confirms both archives are on the channel, derives their URLs from the version, and carries the manifest as this run's own artifact. The npm matrix, the npm publish and the DL build are skipped in that mode; the L4 DL gate and the signed-manifest publish run unchanged, fed from whichever job produced the archives. A normal release is untouched: every new condition is empty on a tag push. --- .github/workflows/release.yml | 85 ++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a74b315..f50d735 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,10 +14,21 @@ on: description: 'Skip GPU tests (alpha cuts — CPU/Metal matrix still gates)' type: boolean default: false + dl_source_run_id: + description: >- + Resume the DL chain for a version whose archives ALREADY uploaded: the + id of the run whose build-dl-flavor succeeded and whose chain failed + after it. Runs only the L4 DL gate and the signed-manifest publish, + against that run's archives and manifest. The pack is deterministic + and the channel paths are write-once, so this is the one way to + finish such a version. Empty for a normal release. + type: string + default: '' jobs: build-and-test: name: Build & Test ${{ matrix.package }} + if: inputs.dl_source_run_id == '' runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -362,6 +373,7 @@ jobs: # ubuntu-24.04 runners building inside an ubuntu:22.04 container, # so the glibc floor survives the runner-image move. runs-on: ubuntu-22.04 + if: inputs.dl_source_run_id == '' outputs: uploaded: ${{ steps.upload.outputs.uploaded }} version: ${{ steps.pack.outputs.version }} @@ -569,10 +581,71 @@ jobs: # assertion → the full model-matrix inference suites. Strictly additive # to the rig's two existing consumers: new script, new JOB_NAME, zero # edits to run-gpu-tests.sh or existing job definitions. + # Resume: the archives and manifest of an EARLIER run of this same version, + # whose DL chain failed after its upload. The write-once paths refuse a second + # upload and the pack is deterministic, so the uploaded bytes ARE the pack; + # what is missing is the L4 gate and the signing act, which run below exactly + # as in a normal release, fed from this job instead of build-dl-flavor. The + # manifest artifact carries the file names and hashes; the URLs derive from + # the version, which must be the version this checkout declares. + dl-source: + name: DL resume source (archives from an earlier run) + if: github.event_name == 'workflow_dispatch' && inputs.dl_source_run_id != '' + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + version: ${{ steps.src.outputs.version }} + archive_url: ${{ steps.src.outputs.archive_url }} + archive_sha256: ${{ steps.src.outputs.archive_sha256 }} + runtime_url: ${{ steps.src.outputs.runtime_url }} + runtime_sha256: ${{ steps.src.outputs.runtime_sha256 }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download the source run's manifest artifact + env: + GH_TOKEN: ${{ github.token }} + run: gh run download "${{ inputs.dl_source_run_id }}" -R "${{ github.repository }}" -n dl-manifest -D dl-manifest + + - name: Resolve the archives the manifest names; refuse a version mismatch or a published manifest + id: src + run: | + set -o pipefail + VERSION=$(node -p "require('./package.json').version") + M=dl-manifest/manifest.json + MV=$(jq -r .version "$M") + test "$MV" = "$VERSION" || { echo "::error::manifest is for $MV but this checkout is $VERSION"; exit 1; } + BASE="https://apps.lloyal.ai/v1/binaries/lloyal.node/$VERSION" + A=$(jq -r .archive.file "$M"); R=$(jq -r .runtimeArchive.file "$M") + for f in "$A" "$R"; do + curl -sfI "$BASE/$f" > /dev/null || { echo "::error::$BASE/$f is not on the channel — nothing to resume"; exit 1; } + done + if curl -sfI "$BASE/linux-x64-dl.manifest.json" > /dev/null; then + echo "::error::a signed manifest is already published for $VERSION"; exit 1 + fi + { + echo "version=$VERSION" + echo "archive_url=$BASE/$A" + echo "archive_sha256=$(jq -r .archive.sha256 "$M")" + echo "runtime_url=$BASE/$R" + echo "runtime_sha256=$(jq -r .runtimeArchive.sha256 "$M")" + } >> "$GITHUB_OUTPUT" + + # publish-dl downloads `dl-manifest` from THIS run, whichever job made it. + - name: Carry the manifest as this run's artifact + uses: actions/upload-artifact@v4 + with: + name: dl-manifest + path: dl-manifest/manifest.json + retention-days: 1 + gpu-tests-dl: name: GPU Tests DL (L4, path-2 consumer) - needs: build-dl-flavor - if: needs.build-dl-flavor.outputs.uploaded == 'true' + needs: [build-dl-flavor, dl-source] + if: ${{ !cancelled() && (needs.build-dl-flavor.outputs.uploaded == 'true' || needs.dl-source.result == 'success') }} runs-on: ubuntu-latest permissions: contents: read @@ -636,15 +709,15 @@ jobs: GCP_REGION: ${{ secrets.GCP_REGION }} GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }} JOB_NAME: lloyal-gpu-test-dl - EXTRA_ENV_VARS: "LLOYAL_PACK_URL=${{ needs.build-dl-flavor.outputs.archive_url }},LLOYAL_PACK_SHA256=${{ needs.build-dl-flavor.outputs.archive_sha256 }},LLOYAL_RUNTIME_URL=${{ needs.build-dl-flavor.outputs.runtime_url }},LLOYAL_RUNTIME_SHA256=${{ needs.build-dl-flavor.outputs.runtime_sha256 }}" + EXTRA_ENV_VARS: "LLOYAL_PACK_URL=${{ needs.build-dl-flavor.outputs.archive_url || needs.dl-source.outputs.archive_url }},LLOYAL_PACK_SHA256=${{ needs.build-dl-flavor.outputs.archive_sha256 || needs.dl-source.outputs.archive_sha256 }},LLOYAL_RUNTIME_URL=${{ needs.build-dl-flavor.outputs.runtime_url || needs.dl-source.outputs.runtime_url }},LLOYAL_RUNTIME_SHA256=${{ needs.build-dl-flavor.outputs.runtime_sha256 || needs.dl-source.outputs.runtime_sha256 }}" # The publish act: sign + create-once the manifest — until this runs, # the uploaded archives are unverifiable ⇒ effectively unpublished. Any # red job upstream ⇒ no signed manifest for this version; npm unaffected. publish-dl: name: Publish DL manifest (signed) - needs: [build-dl-flavor, gpu-tests-dl] - if: needs.build-dl-flavor.outputs.uploaded == 'true' + needs: [build-dl-flavor, dl-source, gpu-tests-dl] + if: ${{ !cancelled() && needs.gpu-tests-dl.result == 'success' && (needs.build-dl-flavor.outputs.uploaded == 'true' || needs.dl-source.result == 'success') }} runs-on: ubuntu-latest steps: # Explicit path: correctness here otherwise rests on two chained @@ -666,7 +739,7 @@ jobs: # would feed empty input to the trailing jq, which exits 0, and # the SIGNING act would report success without publishing. set -o pipefail - VERSION="${{ needs.build-dl-flavor.outputs.version }}" + VERSION="${{ needs.build-dl-flavor.outputs.version || needs.dl-source.outputs.version }}" jq -n --arg v "$VERSION" --slurpfile m dl-manifest/manifest.json \ '{version: $v, platform: "linux-x64-dl", manifest: $m[0]}' \ | curl -sfS -X POST https://api.lloyal.ai/v1/binaries/publish \