From 47a2a4ab0e8ee63eb82bafdd3c870558dd5f4de9 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 15:57:40 -0400 Subject: [PATCH 01/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20reference=20discovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of five slices splitting the Agent Skills feature for review. This one adds the layer with no I/O in it at all: the types, the validation, and the projection from a resolved AI Config to the skills it references. - `skill_refs(config)` projects a config's `skills` array into `list[SkillReference]`. Pure — no client, no store, no network, no telemetry. - `Skill` and `SkillReference` are frozen dataclasses, exported from the package root. `Skill.content` is `bytes` — the verified verbatim bytes LaunchDarkly delivered, exactly what was hashed. Skills are opaque byte buffers by construction: the SDK never parses, decodes, or interprets skill content anywhere. `content_hash` is the sha256 (lowercase hex) over those bytes, and the optional display metadata comes from LaunchDarkly, never from the content. - `parse_ai_config` now validates the optional `skills` array and fails closed on a malformed one. Key grammar, length bound, and the version predicate live in `types_validation.py` as one canonical rejection reason, so every layer added on top rejects a key for the same stated reason. Note one behaviour change for existing users: `parse_ai_config` fails closed on a `skills` value that is not a list of `{key, version}` objects, where before any value parsed and was ignored. A variation carrying its own differently shaped `skills` field must rename it before upgrading. The two layers that follow — retrieval through a store seam, and materialization onto disk — are separate slices. `agents.md` names all three and the one-way dependencies between them. Testing: `uv run pytest` → 1048 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 41 ++++++ packages/client/agents.md | 79 +++++++++- .../src/launchdarkly_ai_server/__init__.py | 9 ++ .../src/launchdarkly_ai_server/skills.py | 72 +++++++++ .../src/launchdarkly_ai_server/types.py | 42 +++++- .../types_validation.py | 76 +++++++++- packages/client/tests/test_schema.py | 103 ++++++++++++- packages/client/tests/test_skills.py | 139 ++++++++++++++++++ uv.lock | 16 +- 9 files changed, 562 insertions(+), 15 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills.py create mode 100644 packages/client/tests/test_skills.py diff --git a/packages/client/README.md b/packages/client/README.md index 8691babd..fbd40045 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -220,6 +220,45 @@ asyncio.run(main()) --- +### Agent Skills + +Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config +variations by reference. This release adds the first layer: discovering which skills a +resolved config references. Retrieving their content and materializing them onto disk follow. + +```python +import asyncio + +from launchdarkly_ai_server import init_client, inspect_config, skill_refs + +async def main(): + await init_client() + + info = await inspect_config("doc-agent", {"kind": "user", "key": "user-123"}) + refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] + + for ref in refs: + print(ref.key, ref.version) + +asyncio.run(main()) +``` + +**`skills` is now a validated field.** Config parsing fails closed on a `skills` value that +is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an +integer ≥ 1): the whole variation is rejected, `inspect_config` returns `config: None`, and +`extract_variation` raises. A variation that previously carried its own custom `skills` +field of a different shape must rename it before upgrading. + +| Export | Description | +|---|---| +| `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | + +> `Skill.content` is `bytes` — the verified verbatim bytes LaunchDarkly delivered, exactly +> what was hashed. The SDK never parses or interprets them; if you want the frontmatter, +> decode and parse the content on your side. + +--- + ### Utility Helpers ```python @@ -250,3 +289,5 @@ All types are exported from this package. Handler packages import them from here | `GraphNode` / `GraphEdge` | A dataclass node (`.key`, `.config`, `.meta`, `.edges`, `.is_terminal`) and a dataclass directed edge (`.key`, `.source_key`, `.target_key`, `.handoff`) | | `ProviderGraphResponse` | A dataclass returned by `graph(...).invoke()`: `.response`, `.usage`, `.judge_results` | | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | +| `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | +| `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | diff --git a/packages/client/agents.md b/packages/client/agents.md index 61d99cf0..8b55f173 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -27,7 +27,8 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/tracking.py` | `execute_and_track`, `execute_and_stream`, `wrap_tool_handlers`, `parse_usage` | | `src/launchdarkly_ai_server/graph.py` | `graph()`, `resolve_graph()`, `GraphInstance` | | `src/launchdarkly_ai_server/types.py` | All shared Python types — `AiConfigRep`, `ProviderHandler`, `LDContext`, `NativeTool`, etc. | -| `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape | +| `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | +| `src/launchdarkly_ai_server/skills.py` | Agent Skills — `skill_refs`, the projection of a config's `skills` array into typed references | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -53,6 +54,7 @@ from launchdarkly_ai_server import ( TrackData, UsageDict, HandlerResult, HandlerStreamEvent, StreamEvent, StreamChunkEvent, StreamDoneEvent, ExecuteStreamEvent, ExecuteStreamDoneEvent, VariationMeta, InitClientOptions, JudgeResult, ParseResult, ParseSuccess, ParseFailure, + Skill, SkillReference, ) # Utilities @@ -69,6 +71,9 @@ from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_t # Entry points from launchdarkly_ai_server import config, graph, resolve_graph + +# Agent Skills +from launchdarkly_ai_server import skill_refs ``` When adding a new export, add it to `__init__.py`'s imports and `__all__`. Handler packages must never import from sub-paths (e.g. `launchdarkly_ai_server.client`). @@ -155,6 +160,32 @@ This is an OTel context value, not W3C baggage, so the id does not leak onto out --- +## Agent Skills + +Versioned `SKILL.md` documents attached to AI Config variations by reference, retrieved +through an injectable store, and materialized onto disk for agent runtimes to discover. +Three layers, in increasing order of blast radius. Only the first is implemented here: + +1. **Reference discovery** — `skill_refs(config)` projects the config's `skills` array into + typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. + Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one + malformed reference fails the whole config parse. +2. **Content accessors** — reading skill content through a store seam. +3. **Materialization** — writing skills onto disk under a manifest. + +### Security posture — do not relax any of this + +- **Skill content is an opaque byte buffer.** `Skill.content` is `bytes` — the verified + verbatim bytes LaunchDarkly delivered, exactly what was hashed. This SDK never parses, + decodes, or interprets them anywhere: not in the integrity path, not in an accessor, not + during materialization. Consumers who want frontmatter parse it themselves. +- **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the + single canonical explanation, so the config parser and the reference projection reject a + key for the same stated reason — and so does every layer added later. A silently + shortened projection is not acceptable: every dropped entry is logged. + +--- + ## OTel Setup The core client owns all OTel initialization. `init_client()` configures a `TracerProvider` with `ConversationIdSpanProcessor` and a `BatchSpanProcessor` plus an OTLP HTTP exporter when the optional OTel packages are installed. @@ -230,6 +261,43 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b --- +## Dependencies + +Tier 0, so the runtime surface is deliberately tiny: **one** hard dependency, and everything else either an optional extra, resolved dynamically at runtime, or dev-only. Nothing here may grow without a reason recorded in this table. + +### Runtime (`[project] dependencies`) + +| Package | Why | +|---|---| +| `opentelemetry-api>=1.25` | The tracer/span API used on every instrumented path (`tracking.py`, `graph.py`, `content.py`, `conversation.py`, `utils.py`). API-only — the *SDK* half is an optional extra, so a consumer that never configures OTel gets no-op spans rather than an `ImportError`. `conversation.py` imports `opentelemetry.sdk.trace.SpanProcessor` under `TYPE_CHECKING` only, for exactly this reason. | + +There is deliberately **no** `python-dotenv` here: `lifecycle.py` reads `os.environ` directly, so loading a `.env` file is the application's job rather than the SDK's. `python-dotenv` is in the workspace dev group for the examples only. + +### Optional extra (`[project.optional-dependencies] otel`) + +| Package | Why | +|---|---| +| `opentelemetry-sdk>=1.25` | Tracer provider, resources, and the batch span processor, imported inside `_setup_telemetry()` in `lifecycle.py`. Optional so telemetry is opt-in; absent ⇒ a `logger.warning` and no spans, never a raise. | +| `opentelemetry-exporter-otlp-proto-http>=1.25` | OTLP/HTTP span export and its compression enum. Same optionality, same loader. | + +Install with `pip install "launchdarkly-ai-server[otel]"`; see [OTel Setup](#otel-setup) for the endpoint variables. + +### Resolved dynamically, declared nowhere + +| Package | Why | +|---|---| +| `launchdarkly-server-sdk` | The LaunchDarkly server SDK, reached by `importlib.import_module("ldclient")` (falling back to `launchdarkly_server_sdk`) inside `init_client()`'s options path. Undeclared on purpose: the BYOC path (`init_client(client=...)`) targets environments that supply their own client, and a hard dependency would force an unused SDK into every such install. So it is imported late and raises actionably when missing — absent ⇒ a `RuntimeError` naming the `pip install`, and only on the path that needs it. | + +### Dev-only (workspace root `[dependency-groups] dev`) — the ones with a contract attached + +| Package | Why | +|---|---| +| `launchdarkly-server-sdk>=9.0`, and the `otel` extra mirrored (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) | Each dynamically-resolved or optional package is repeated in the dev group so the test suite can import it. Something that is *only* optional would not be installed in this workspace and the tests covering its present-and-working path could not run. | +| `pytest>=8`, `pytest-asyncio>=0.24` | Test runner and the async support the whole suite relies on. `asyncio_mode = "auto"` is set at the workspace root, which is why no test in this package carries an `@pytest.mark.asyncio`. | +| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. | + +--- + ## Common Pitfalls ### 1. Calling `get_client()` before `init_client()` resolves @@ -240,6 +308,14 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b `execute_and_track` expects the handler to return a plain `dict` with at least `output` and `usage` keys. Do not return a custom class — `parse_usage` and the telemetry pipeline both access dict keys. +### 3. Interpreting skill content anywhere + +`Skill.content` is opaque `bytes` by construction. Do not add a parser, a decoder, or a +convenience accessor that reads meaning into it — no YAML/frontmatter parsing, no +"decode as UTF-8 for display", nothing. The SDK's whole contract is that content is the +verified verbatim byte buffer and nothing more; a consumer who wants structure parses it +on their side of the boundary. + --- ## Adding a New Export @@ -255,3 +331,4 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b - Handler packages must import `LDContext` from `launchdarkly-ai-server` — not directly from any LD SDK. - Do not weaken the `parse_ai_config` validation — handler packages rely on `config` being valid when they receive it. - `parse_usage` must continue to accept `input_tokens/output_tokens`, `inputTokens/outputTokens`, and `input/output` as all existing handlers return one of these variants. +- `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 80b959bc..3faff304 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -37,6 +37,9 @@ resolve_handlers, resolve_tools, ) +from .skills import ( + skill_refs, +) from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -66,6 +69,8 @@ ProviderGraphResponse, ProviderHandler, ProviderResponse, + Skill, + SkillReference, StreamChunkEvent, StreamDoneEvent, StreamEvent, @@ -126,6 +131,8 @@ "ProviderGraphResponse", "ProviderHandler", "ProviderResponse", + "Skill", + "SkillReference", "StreamChunkEvent", "StreamDoneEvent", "StreamEvent", @@ -199,4 +206,6 @@ "graph", "resolve_graph", "GraphInstance", + # skills + "skill_refs", ] diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py new file mode 100644 index 00000000..a3294bd2 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -0,0 +1,72 @@ +""" +Agent Skills — reference discovery. + +Projects the skill references a resolved AI Config carries into typed values. A +pure projection: no network, no client, no store, no telemetry. Retrieving the +content those references point at is a separate layer. +""" + +from __future__ import annotations + +import logging + +from .types import AiConfigRep, SkillReference +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Reference discovery +# --------------------------------------------------------------------------- +def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: + """ + Projects a resolved AI Config's ``skills`` array into typed references. + + A pure projection — no network, no client, no store, no telemetry. Returns + ``[]`` when the config carries no skills. + + A config that came through ``parse_ai_config`` never contains an invalid + entry — parsing fails closed on one. A hand-built dict can, and a silently + shortened projection would leave a caller materializing a skill set it + believes is complete, so every dropped entry is logged. + """ + if not isinstance(config, dict): + return [] + + raw = config.get("skills") + if not isinstance(raw, list): + return [] + + refs: list[SkillReference] = [] + for index, entry in enumerate(raw): + if not isinstance(entry, dict): + logger.warning( + "skills[%d] is not a {key, version} object; it was dropped " + "from the projection", + index, + ) + continue + key = entry.get("key") + version = entry.get("version") + # Branch on the TypeGuard predicate (not the reason string) so the type + # checker narrows ``key`` to ``str`` for the reference below. + if not is_valid_skill_key(key): + logger.warning( + "skills[%d].key %s; it was dropped from the projection", + index, + skill_key_rejection_reason(key), + ) + elif not is_valid_skill_version(version): + logger.warning( + "skills[%d].version must be an integer >= 1; it was dropped " + "from the projection", + index, + ) + else: + refs.append(SkillReference(key=key, version=version)) + return refs diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 57f81343..4249d5cb 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -92,7 +92,8 @@ class Message: AiConfigRep = dict[str, Any] """ Raw AI config dict as returned by ``parse_ai_config``. Fields include -``model``, ``provider``, and at least one of ``instructions`` / ``messages``. +``model``, ``provider``, at least one of ``instructions`` / ``messages``, and an +optional ``skills`` array of ``{key, version}`` references (see ``skill_refs``). """ VariationMeta = dict[str, Any] @@ -412,6 +413,45 @@ class ProviderGraphResponse: """Results from a graph-level judge, if configured.""" +# --------------------------------------------------------------------------- +# Agent Skills +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SkillReference: + """A version-pinned pointer to a skill, as attached to an AI Config variation.""" + + key: str + """Immutable skill key — ``^[a-z0-9][a-z0-9-]*$``, at most 256 characters.""" + version: int + """Immutable skill version — an integer >= 1.""" + + +@dataclass(frozen=True) +class Skill: + """ + A single verbatim ``SKILL.md`` document. + + Only ever constructed after integrity verification passes, so ``content`` + holds the exact byte sequence LaunchDarkly delivered and ``content_hash`` + is its sha256. Instances are immutable. + """ + + key: str + version: int + content: bytes + """The verified verbatim bytes, exactly as LaunchDarkly delivered and + hashed them. Opaque to this SDK: no encoding is claimed and nothing here + ever parses or interprets them.""" + content_hash: str + """sha256, lowercase hex, over the verbatim bytes of ``content``.""" + name: str | None = None + """Display name from LaunchDarkly metadata; never parsed from the content.""" + description: str | None = None + """Description from LaunchDarkly metadata; never parsed from the content.""" + + # --------------------------------------------------------------------------- # Model / graph options # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/types_validation.py b/packages/client/src/launchdarkly_ai_server/types_validation.py index acdcce42..37cd56bf 100644 --- a/packages/client/src/launchdarkly_ai_server/types_validation.py +++ b/packages/client/src/launchdarkly_ai_server/types_validation.py @@ -1,16 +1,62 @@ from __future__ import annotations -from typing import Any +import re +from typing import Any, TypeGuard from .types import ParseFailure, ParseResult, ParseSuccess _VALID_ROLES = {"user", "assistant", "system"} +SKILL_KEY_GRAMMAR = "^[a-z0-9][a-z0-9-]*$" +""" +The skill key grammar, as a string, so every message that has to explain a +rejection quotes the rule rather than restating it. Tightening the pattern below +then cannot leave an error message describing the old grammar. +""" + +_SKILL_KEY_PATTERN = re.compile(r"\A[a-z0-9][a-z0-9-]*\Z") +""" +``SKILL_KEY_GRAMMAR``, anchored with ``\\A``/``\\Z`` rather than ``^``/``$`` +because ``$`` also matches immediately before a trailing newline, which would +let ``"pdf-extraction\\n"`` through as a directory name. +""" + +SKILL_KEY_MAX_LENGTH = 256 +"""Longest key the data model permits. Note that no mainstream filesystem allows +a 256-byte path component, so ``write_skills`` applies a tighter bound of its own.""" + def _is_object(v: Any) -> bool: return isinstance(v, dict) +def skill_key_rejection_reason(key: Any) -> str | None: + """ + Why *key* is not a valid skill key, or ``None`` when it is. + + The canonical explanation, so the config parser, the filesystem layer and + the reference projection all reject a key for the same stated reason. + ``is_valid_skill_key`` is this predicate with the reason discarded. + """ + if not isinstance(key, str): + return "must be a string" + if len(key) > SKILL_KEY_MAX_LENGTH: + return f"must be at most {SKILL_KEY_MAX_LENGTH} characters" + if _SKILL_KEY_PATTERN.match(key) is None: + return f"must match {SKILL_KEY_GRAMMAR}" + return None + + +def is_valid_skill_key(key: Any) -> TypeGuard[str]: + """Skill keys are untrusted input everywhere they appear — validate every time.""" + return isinstance(key, str) and skill_key_rejection_reason(key) is None + + +def is_valid_skill_version(version: Any) -> TypeGuard[int]: + """Skill versions are integers >= 1. ``bool`` is not an acceptable integer.""" + return isinstance(version, int) and not isinstance(version, bool) and version >= 1 + + def _parse_tool(raw: Any, key: str) -> str | None: """Returns an error message string or ``None`` on success.""" if not _is_object(raw): @@ -24,6 +70,28 @@ def _parse_tool(raw: Any, key: str) -> str | None: return None +def _parse_skills(raw: Any) -> str | None: + """ + Validates the optional ``skills`` array. Returns an error message or ``None``. + + Fail closed: a malformed reference makes the whole config malformed, because + an SDK that silently dropped a bad reference would materialize a partial + skill set without telling anyone. + """ + if not isinstance(raw, list): + return "skills must be an array of {key, version} objects" + + for index, entry in enumerate(raw): + if not _is_object(entry): + return f"skills[{index}] must be an object with key and version" + key_rejection = skill_key_rejection_reason(entry.get("key")) + if key_rejection is not None: + return f"skills[{index}].key {key_rejection}" + if not is_valid_skill_version(entry.get("version")): + return f"skills[{index}].version must be an integer >= 1" + return None + + def parse_ai_config(raw: Any) -> ParseResult: """ Validates a raw LaunchDarkly flag variation as an ``AiConfigRep``. @@ -88,4 +156,10 @@ def parse_ai_config(raw: Any) -> ParseResult: error={"message": "outputFormat must be an object (JSON Schema)"}, ) + skills = raw.get("skills") + if skills is not None: + err = _parse_skills(skills) + if err: + return ParseFailure(success=False, error={"message": err}) + return ParseSuccess(success=True, data=raw) diff --git a/packages/client/tests/test_schema.py b/packages/client/tests/test_schema.py index fad75cf0..3be146d3 100644 --- a/packages/client/tests/test_schema.py +++ b/packages/client/tests/test_schema.py @@ -1,7 +1,8 @@ -""" -Tests for §3.5 parse_ai_config (AiConfig validation). -Reference: TESTING.md §3.5 -""" +"""Tests for ``parse_ai_config`` — AI Config variation validation.""" + +from typing import Any + +import pytest from launchdarkly_ai_server import parse_ai_config @@ -81,3 +82,97 @@ def test_output_format_accepted(self) -> None: raw["outputFormat"] = {"type": "object", "properties": {}} result = parse_ai_config(raw) assert result.success is True + + +class TestParseAiConfigSkills: + """ + Fail-closed validation of the optional ``skills`` array. + """ + + def _base(self, **extra: Any) -> dict[str, Any]: + raw: dict[str, Any] = { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + } + raw.update(extra) + return raw + + def test_absent_skills_is_valid(self) -> None: + assert parse_ai_config(self._base()).success is True + + def test_empty_skills_is_valid(self) -> None: + assert parse_ai_config(self._base(skills=[])).success is True + + def test_valid_entries_accepted(self) -> None: + raw = self._base(skills=[{"key": "pdf-extraction", "version": 2}]) + result = parse_ai_config(raw) + assert result.success is True + assert result.data["skills"] == [{"key": "pdf-extraction", "version": 2}] + + def test_multiple_valid_entries_accepted(self) -> None: + raw = self._base( + skills=[{"key": "a", "version": 1}, {"key": "b-2", "version": 10}] + ) + assert parse_ai_config(raw).success is True + + def test_key_at_length_bound_accepted(self) -> None: + raw = self._base(skills=[{"key": "a" * 256, "version": 1}]) + assert parse_ai_config(raw).success is True + + @pytest.mark.parametrize("bad_skills", ["pdf", {"key": "a"}, 3, True]) + def test_non_array_skills_fails(self, bad_skills: Any) -> None: + assert parse_ai_config(self._base(skills=bad_skills)).success is False + + @pytest.mark.parametrize("entry", ["pdf-extraction", 1, None, ["a", 1]]) + def test_non_object_entry_fails(self, entry: Any) -> None: + assert parse_ai_config(self._base(skills=[entry])).success is False + + @pytest.mark.parametrize("bad_key", [None, 1, True, {"a": 1}, ["a"]]) + def test_missing_or_non_string_key_fails(self, bad_key: Any) -> None: + raw = self._base(skills=[{"key": bad_key, "version": 1}]) + assert parse_ai_config(raw).success is False + + def test_absent_key_fails(self) -> None: + assert parse_ai_config(self._base(skills=[{"version": 1}])).success is False + + @pytest.mark.parametrize( + "bad_key", + [ + "", + "Evil", + "-leading-dash", + ".hidden", + "_underscore", + "has space", + "a/b", + "a\\b", + "../escape", + "trailing-space ", + "under_score", + "a" * 257, + ], + ) + def test_pattern_and_length_violations_fail(self, bad_key: str) -> None: + raw = self._base(skills=[{"key": bad_key, "version": 1}]) + assert parse_ai_config(raw).success is False + + @pytest.mark.parametrize("bad_version", [0, -1, 2.5, "2", None, True, [1]]) + def test_invalid_version_fails(self, bad_version: Any) -> None: + raw = self._base(skills=[{"key": "a", "version": bad_version}]) + assert parse_ai_config(raw).success is False + + def test_absent_version_fails(self) -> None: + assert parse_ai_config(self._base(skills=[{"key": "a"}])).success is False + + def test_one_bad_entry_fails_the_whole_config(self) -> None: + raw = self._base( + skills=[{"key": "good", "version": 1}, {"key": "../bad", "version": 1}] + ) + assert parse_ai_config(raw).success is False + + def test_error_message_mentions_skills(self) -> None: + raw = self._base(skills=[{"key": "../bad", "version": 1}]) + result = parse_ai_config(raw) + assert result.success is False + assert "skills" in result.error["message"] diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py new file mode 100644 index 00000000..38ca34f2 --- /dev/null +++ b/packages/client/tests/test_skills.py @@ -0,0 +1,139 @@ +""" +Agent Skills — reference discovery: the types and the projection from a +resolved AI Config. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +from typing import Any + +import pytest + +from launchdarkly_ai_server import ( + Skill, + SkillReference, + skill_refs, +) + +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + + +def _hash(content: str) -> str: + """sha256, lowercase hex, over verbatim utf-8 bytes.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", + version: int = 1, + content: str = SKILL_BODY, + content_hash: str | None = None, +) -> Skill: + """Build a verified-shaped Skill directly (bypasses the accessors).""" + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=content_hash if content_hash is not None else _hash(content), + ) + + +class TestSkillTypes: + """Immutability and optional metadata.""" + + def test_skill_reference_is_immutable(self) -> None: + ref = SkillReference(key="pdf-extraction", version=2) + with pytest.raises(dataclasses.FrozenInstanceError): + ref.version = 3 # type: ignore[misc] + + def test_skill_is_immutable(self) -> None: + skill = _skill() + with pytest.raises(dataclasses.FrozenInstanceError): + skill.content = b"tampered" # type: ignore[misc] + + def test_skill_content_is_bytes(self) -> None: + """Content is the verified verbatim bytes — opaque, never text.""" + skill = _skill() + assert isinstance(skill.content, bytes) + assert skill.content == SKILL_BODY.encode("utf-8") + + def test_skill_carries_optional_metadata(self) -> None: + skill = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash=_hash(SKILL_BODY), + name="A Skill", + description="does things", + ) + assert skill.name == "A Skill" + assert skill.description == "does things" + + def test_skill_metadata_defaults_to_none(self) -> None: + skill = _skill() + assert skill.name is None + assert skill.description is None + + +class TestSkillRefs: + """Pure projection of the config's skills array.""" + + def _config(self, **extra: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "hi", + } + base.update(extra) + return base + + def test_absent_skills_returns_empty_list(self) -> None: + assert skill_refs(self._config()) == [] + + def test_empty_skills_returns_empty_list(self) -> None: + assert skill_refs(self._config(skills=[])) == [] + + def test_returns_typed_references_in_order(self) -> None: + config = self._config( + skills=[{"key": "a", "version": 1}, {"key": "b", "version": 3}] + ) + refs = skill_refs(config) + assert refs == [ + SkillReference(key="a", version=1), + SkillReference(key="b", version=3), + ] + assert all(isinstance(r, SkillReference) for r in refs) + + def test_dropped_entries_are_logged(self, caplog: pytest.LogCaptureFixture) -> None: + """A shortened projection is never silent. + + ``parse_ai_config`` fails the whole config closed on a malformed + reference, so a config that reached here through it cannot contain one. + A hand-built dict can, and feeding the shortened list to + ``write_skills`` would prune the dropped skill's on-disk copy — so the + drop is observable rather than silent. + """ + config = self._config( + skills=[ + {"key": "good", "version": 1}, + {"key": "bad", "version": 0}, + {"key": "Bad-Key", "version": 1}, + "not-an-object", + ] + ) + + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills"): + refs = skill_refs(config) + + assert refs == [SkillReference(key="good", version=1)] + assert len(caplog.records) == 3 + # The body is never echoed, and neither is the invalid key. + assert all("skills[" in r.getMessage() for r in caplog.records) + + def test_requires_no_client_or_store(self, mock_ld_client: Any) -> None: + """No store configured, no client initialized — still a pure projection.""" + refs = skill_refs(self._config(skills=[{"key": "a", "version": 2}])) + assert refs == [SkillReference(key="a", version=2)] + mock_ld_client.track.assert_not_called() diff --git a/uv.lock b/uv.lock index de575a71..7d93a3cd 100644 --- a/uv.lock +++ b/uv.lock @@ -790,7 +790,7 @@ wheels = [ [[package]] name = "launchdarkly-ai-claude-agents" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/claude-agents" } dependencies = [ { name = "anthropic" }, @@ -809,7 +809,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-claude-messages" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/claude-messages" } dependencies = [ { name = "anthropic" }, @@ -826,7 +826,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-agents" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/langchain-agents" } dependencies = [ { name = "langchain-core" }, @@ -845,7 +845,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-messages" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/langchain-messages" } dependencies = [ { name = "langchain-core" }, @@ -862,7 +862,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-agents" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/openai-agents" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -881,7 +881,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-messages" -version = "0.1.1" +version = "0.1.4" source = { editable = "packages/openai-messages" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -898,7 +898,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-python" -version = "0.1.1" +version = "0.1.3" source = { editable = "packages/ai" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -918,7 +918,7 @@ provides-extras = ["otel"] [[package]] name = "launchdarkly-ai-server" -version = "0.1.1" +version = "0.1.3" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, From 27ef12f85faa2790ed6ecac8c23d78bd7a2304ad Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:04:25 -0400 Subject: [PATCH 02/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20retrieval=20through=20an=20injectable=20store?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second of five slices. Adds the layer that turns a reference into content: an injectable store seam, integrity verification of everything it serves, and a body-free telemetry seam for the failures. - `get_skill(key, *, version=None)` returns one verified skill, or None. - `get_skills(refs)` is the batch form, accepting references and bare keys. - `all_skills()` returns every verified skill the store holds, one per key. - `SkillStore` is the structural interface content arrives through — `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional `add_listener(kind, fn)` — configured with `init_client(options={"skillStore": store})`. `InMemorySkillStore` ships for local development and testing. A delivery transport drops in behind the same seam with no public API change. Store data is untrusted. Key and version are revalidated, size is bounded, and the sha256 of the verbatim bytes must match the delivered `contentHash`; anything that does not verify is withheld and treated as missing, so no unverified content is ever returned. The wire object delivers content as a JSON string; the UTF-8 encode happens exactly once, inside verification, and the `Skill` handed to user code carries the verified verbatim bytes (`Skill.content: bytes`) — the exact byte sequence that was hashed, never a re-derived value. Content carrying an unpaired surrogate has no UTF-8 encoding at all and is withheld too — `str.encode` is called strictly, never with an error handler that would fabricate bytes a hash comparison could then accept. `verified_bytes` also accepts already-bytes content, hashing it directly, for the pre-write re-verification pass a later slice adds. Integrity failures are reported through a private telemetry seam carrying hashes and byte counts only, never the skill body. The two properties copied off the wire, `skill_key` and `expected_hash`, are shape-checked and replaced when malformed, so a hostile store cannot use either one to publish the body through a signal that is otherwise body-free. The default emitter is a no-op: nothing leaves the process in this release, and the three signal names are an allowlist maintained in one section of one module. Version is part of the lookup identity rather than a filter applied to the answer. A delivery payload carries the newest version of every skill plus every version any variation currently pins, so two versions of one key coexist routinely; a seam keyed by key alone would answer a pinned reference with the newest object and then reject it, turning the primary use case into a missing skill. `InMemorySkillStore` holds several versions of a key, `get_object` takes the wanted version, and `version=None` means "the newest you hold". The equality check afterwards is kept as a defense — the store is untrusted, so an answer that is not the version asked for is withheld. `all_objects` returns one entry per key-and-version under keys that are opaque to this SDK; identity is read off each object's own fields. `newest_by_key` is the single place that collapses the result to one object per key. A run that withheld anything now logs a count at WARN. Every individual withholding already records a signal and an error line, but a caller reading logs at WARN saw neither, and a payload where nothing verifies otherwise returns an empty result indistinguishable from "this project has no skills". `SKILL_OBJECT_KIND` is deliberately **not** exported from the package root. It is the string this SDK hands a store, and an adapter maps whatever the transport underneath calls a skill onto it; publishing it would advertise an SDK-side seam value as the wire contract. An adapter that needs to agree with it reaches it through `skills_core`. `MAX_SKILL_CONTENT_BYTES` stays internal for the adjacent reason. Note one behaviour change: `shutdown()` clears the configured skill store along with the client. `init_client` applies `skillStore` on every successful call, even the idempotent ones, which is what lets a lazily auto-initialized client be given a store afterwards. Testing: `uv run pytest` → 1145 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 61 +- packages/client/agents.md | 112 +- .../src/launchdarkly_ai_server/__init__.py | 10 + .../src/launchdarkly_ai_server/lifecycle.py | 42 +- .../src/launchdarkly_ai_server/skills.py | 242 +++- .../src/launchdarkly_ai_server/skills_core.py | 628 ++++++++++ packages/client/tests/conftest.py | 112 ++ packages/client/tests/test_skills.py | 1019 ++++++++++++++++- 8 files changed, 2203 insertions(+), 23 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_core.py diff --git a/packages/client/README.md b/packages/client/README.md index fbd40045..f16a1eb2 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -223,22 +223,46 @@ asyncio.run(main()) ### Agent Skills Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config -variations by reference. This release adds the first layer: discovering which skills a -resolved config references. Retrieving their content and materializing them onto disk follow. +variations by reference. The SDK surfaces which skills a config references and retrieves +their content. Materializing them onto disk, where agent runtimes discover them, follows. ```python import asyncio +import hashlib -from launchdarkly_ai_server import init_client, inspect_config, skill_refs +from launchdarkly_ai_server import ( + init_client, inspect_config, skill_refs, get_skill, get_skills, + InMemorySkillStore, +) + +SKILL_MD = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" async def main(): - await init_client() + # A store supplies skill content. InMemorySkillStore is the dict-backed + # store for local development, testing, and bring-your-own-content use. + store = InMemorySkillStore() + store.put({ + "key": "pdf-extraction", + "version": 2, + "content": SKILL_MD, + # sha256, lowercase hex, over the verbatim utf-8 bytes. Content whose hash + # does not match is withheld, so this is not optional. + "contentHash": hashlib.sha256(SKILL_MD.encode("utf-8")).hexdigest(), + }) + await init_client(options={"skillStore": store}) + # 1. Which skills does this config reference? Pure projection — no I/O. info = await inspect_config("doc-agent", {"kind": "user", "key": "user-123"}) - refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] + refs = skill_refs(info["config"]) # [SkillReference(key='pdf-extraction', version=2)] - for ref in refs: - print(ref.key, ref.version) + # 2. Fetch content. Returns None rather than raising when a skill is unavailable. + skill = await get_skill("pdf-extraction") + if skill is not None: + print(skill.content) + + # 3. Or resolve the config's references in one call. + for s in await get_skills(refs): + print(s.key, s.version) asyncio.run(main()) ``` @@ -249,9 +273,32 @@ integer ≥ 1): the whole variation is rejected, `inspect_config` returns `confi `extract_variation` raises. A variation that previously carried its own custom `skills` field of a different shape must rename it before upgrading. +**Integrity is not optional.** Content is only returned after its sha256 (lowercase hex, +over the verbatim UTF-8 bytes) matches the delivered `contentHash`, its key and version +revalidate, and its size is within 64 KiB. Anything that fails is withheld and treated as +missing — no unverified content ever reaches your code. A retrieval that withheld anything +logs a count at WARN, so a run that resolved nothing is not silent. + +**Versions are selected, not filtered.** A store may hold several versions of one key at +once, because a delivery payload does: the newest version of every skill, plus every +version a variation currently pins. `get_skill("k", version=1)` asks the store for version +1 and gets it even when a newer one is also held. + | Export | Description | |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | +| `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | +| `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | +| `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | + +Configure the store with `init_client(options={"skillStore": store})`. With none configured, +the accessors raise `RuntimeError` explaining what to do. `shutdown()` clears it. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the +SDK — identity is read from each object's own `key` and `version` fields, so a store is free +to key its own map however the transport underneath does. > `Skill.content` is `bytes` — the verified verbatim bytes LaunchDarkly delivered, exactly > what was hashed. The SDK never parses or interprets them; if you want the frontmatter, diff --git a/packages/client/agents.md b/packages/client/agents.md index 8b55f173..f284c425 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -28,7 +28,8 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/graph.py` | `graph()`, `resolve_graph()`, `GraphInstance` | | `src/launchdarkly_ai_server/types.py` | All shared Python types — `AiConfigRep`, `ProviderHandler`, `LDContext`, `NativeTool`, etc. | | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | -| `src/launchdarkly_ai_server/skills.py` | Agent Skills — `skill_refs`, the projection of a config's `skills` array into typed references | +| `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | +| `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -73,9 +74,22 @@ from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_t from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills -from launchdarkly_ai_server import skill_refs +from launchdarkly_ai_server import ( + skill_refs, get_skill, get_skills, all_skills, + SkillStore, InMemorySkillStore, +) ``` +`MAX_SKILL_CONTENT_BYTES` is deliberately *not* among them: it is a local enforcement +bound on content the platform produces, not a value this SDK defines, so exporting it +would semver-lock a number this side does not own. Keep it internal to `skills_core`. + +`SKILL_OBJECT_KIND` is not exported either, for a different reason: it is the string this +SDK hands a store, and a store adapter maps whatever the transport underneath calls a skill +onto it. Publishing it would advertise an SDK-side seam value as the wire contract — a claim +this side cannot make, and hard to walk back once a caller depends on it. An adapter that +needs to agree with it reaches it through `launchdarkly_ai_server.skills_core`. + When adding a new export, add it to `__init__.py`'s imports and `__all__`. Handler packages must never import from sub-paths (e.g. `launchdarkly_ai_server.client`). --- @@ -170,20 +184,103 @@ Three layers, in increasing order of blast radius. Only the first is implemented typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one malformed reference fails the whole config parse. -2. **Content accessors** — reading skill content through a store seam. +2. **Content accessors** — `get_skill`, `get_skills`, `all_skills` read through the + `SkillStore` seam. Configure a store with + `init_client(options={"skillStore": store})`; with none configured the accessors raise + an actionable `RuntimeError`. A delivery transport can be added behind the seam + without touching the public API. 3. **Materialization** — writing skills onto disk under a manifest. +### The store seam, and why version is part of the lookup + +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional +`add_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied +to the answer, and that is load-bearing: a delivery payload carries the newest version of +every skill *plus* every version any variation currently pins, so two versions of one key +coexist routinely. A seam keyed by key alone would answer a pinned reference with the newest +object, and the caller would then have to reject it — turning the primary use case, a +version-pinned attachment, into a missing skill. `version=None` asks for the newest held. + +The equality check in `resolve_from_store` stays, now as a **defense** rather than as the +selection mechanism: the store is untrusted, so an answer that is not the version asked for +is withheld. + +`all_objects` returns one entry per `(key, version)` under keys that are **opaque** to this +SDK. Do not parse them and do not assume one per skill key; identity is read off each +object's own `key` and `version`, which are revalidated anyway. `newest_by_key` is the +one place that collapses the result to one object per key, because both whole-store +consumers need it — `all_skills`, since a list holding two versions of one key is not a set +of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. + ### Security posture — do not relax any of this +Store data is **untrusted input**; the transport is not part of the trust boundary. + - **Skill content is an opaque byte buffer.** `Skill.content` is `bytes` — the verified - verbatim bytes LaunchDarkly delivered, exactly what was hashed. This SDK never parses, - decodes, or interprets them anywhere: not in the integrity path, not in an accessor, not - during materialization. Consumers who want frontmatter parse it themselves. + verbatim bytes, exactly what was hashed. The wire object delivers content as a JSON + string; the UTF-8 encode happens once, during verification, and from then on the SDK + never parses, decodes, or interprets the bytes anywhere: not in the integrity path, not + in an accessor, not during materialization. Consumers who want frontmatter parse it + themselves. +- **Integrity is mandatory and doubled, through one implementation.** Every raw object is + verified at the accessor boundary (key pattern and length, integer version >= 1, content + at most 64 KiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) + and the hash is re-verified immediately before a write, both through + `skills_core.verified_bytes`, so the integrity signal's property set cannot depend on + which layer caught the defect. A `Skill` is only ever constructed from content that + passed. Nothing unverified reaches user code. +- **`contentHash` is required.** An object without one is withheld, not accepted on trust. + A payload built before the field is populated therefore yields nothing, which is why a + withholding run logs a run-level count at WARN — an empty result would otherwise be + indistinguishable from "this project has no skills". +- **No unencodable string ever reaches an encode.** `json.loads` turns a `\ud800` escape + into an unpaired surrogate with no UTF-8 representation; every `.encode("utf-8")` site + treats that as a verification failure. Never reach for `errors="surrogatepass"` — + fabricating bytes could satisfy the hash comparison. +- **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` + come off the wire, so a store could put the skill body in either; both are shape-checked + and redacted before they reach a signal or a log line. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently shortened projection is not acceptable: every dropped entry is logged. +### Telemetry seam + +Skills telemetry goes through a private emitter with one method, +`record(signal, properties)`, whose default implementation is a **no-op** — nothing leaves +the process in this release. `client.track()` is deliberately *not* used: it needs an LD +context, spends the customer's event volume, lands in their data export, and is silenced by +offline mode. No LD context is involved anywhere in this feature. + +Exactly three signals exist, and the list is an **allowlist, not a floor**: + +| Signal | When | Properties | +|---|---|---| +| `AgentControl Skill Integrity Failure` | any hash/size/shape verification failure | `skill_key`, `version?`, `expected_hash?`, `observed_hash?`, `language` | +| `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | +| `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | + +The last two belong to the materialization layer and have no caller yet; they live here +with the first so the allowlist is one section of one file rather than three sites to audit. + +`AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` +were considered and **deliberately excluded from SDK emission** — both are observable +server-side. Do not add them. The skill body never appears in a signal, a log line, or +an error message, and no signal carries a filesystem path. An emitter that raises is caught +and logged; it never fails the operation. + +Module state lives in `skills_core.py`, so there is exactly one store and one emitter +however the feature is entered. All three signals are emitted from the `record_*` functions +next to the seam there — nothing outside that module calls `emit`, so the allowlist is +enforced in one place. + +The injection path is deliberately narrower than the state's location: `skills.py` owns +`_set_store`, `_set_emitter_for_testing` and `_clear_state`, which delegate to +`skills_core`. `init_client` and `shutdown` use those, tests inject through those +(`skills._set_store(store)` is the same setter `init_client` uses), and neither should +reach into `skills_core` directly. + --- ## OTel Setup @@ -332,3 +429,6 @@ on their side of the boundary. - Do not weaken the `parse_ai_config` validation — handler packages rely on `config` being valid when they receive it. - `parse_usage` must continue to accept `input_tokens/output_tokens`, `inputTokens/outputTokens`, and `input/output` as all existing handlers return one of these variants. - `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. +- Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. +- Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. +- Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 3faff304..02c858f2 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -38,8 +38,13 @@ resolve_tools, ) from .skills import ( + InMemorySkillStore, + all_skills, + get_skill, + get_skills, skill_refs, ) +from .skills_core import SkillStore from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -208,4 +213,9 @@ "GraphInstance", # skills "skill_refs", + "get_skill", + "get_skills", + "all_skills", + "SkillStore", + "InMemorySkillStore", ] diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 8969f2ea..28b7e81e 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -6,6 +6,7 @@ import os from typing import Any +from . import skills from .types import InitClientOptions logger = logging.getLogger(__name__) @@ -130,13 +131,43 @@ async def init_client( - Pass *client* directly (BYOC) to skip the LaunchDarkly Python SDK path. - Otherwise, reads ``LD_SDK_KEY`` from env or ``options['sdkKey']``. + - ``options['skillStore']`` configures the store the Agent Skills accessors + read from. Absent by default, in which case they raise an actionable error. + + This function is idempotent for the client singleton: a second call returns + the existing client without re-initializing, and every option is ignored — + **except** ``skillStore``, which is applied on every successful call. That + asymmetry is deliberate, and it is what lets a client that was lazily + auto-initialized, or initialized without a store, be given one afterwards. + A ``skillStore`` of ``None`` (or absent) never clears an already-configured + store; use ``shutdown()`` for that. The store is installed only once + initialization has succeeded: a call that raises leaves no global state + behind, so a failed init cannot leave the skill accessors working against a + store the application believes was never installed. Returns the initialized ``LDClientInterface`` instance. """ - global _client - opts = options or {} + ld_client = await _resolve_client(opts, client) + + # The single success point: every path that raises returns before here, so + # "installed only on success" is one statement rather than a copy per exit. + skill_store = opts.get("skillStore") + if skill_store is not None: + skills._set_store(skill_store) + return ld_client + + +async def _resolve_client(opts: InitClientOptions, client: Any) -> Any: + """ + Returns the singleton client, initializing it on first call. + + Split from ``init_client`` so that function has exactly one success point to + hang the ``skillStore`` carve-out on. + """ + global _client + # Idempotent — if already initialized, return the existing client if _client is not None: return _client @@ -198,12 +229,18 @@ async def shutdown() -> None: """ Shuts down the singleton client. Idempotent — safe to call multiple times even if the client was never initialized or already shut down. + + Also clears the configured skill store (and telemetry emitter): after a + shutdown, re-pass ``skillStore`` to the next ``init_client`` if the skill + accessors should keep working. """ global _client, _tracer_provider local_client = _client local_provider = _tracer_provider + skills._clear_state() + # Null the singleton before any awaits so a second call is a no-op _client = None _tracer_provider = None @@ -240,6 +277,7 @@ def _reset_for_testing() -> None: global _client, _tracer_provider _client = None _tracer_provider = None + skills._clear_state() async def inspect_config( diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index a3294bd2..eb3946c2 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -1,16 +1,43 @@ """ -Agent Skills — reference discovery. +Agent Skills — reference discovery and content accessors. -Projects the skill references a resolved AI Config carries into typed values. A -pure projection: no network, no client, no store, no telemetry. Retrieving the -content those references point at is a separate layer. +The public retrieval surface: projecting the skill references a resolved AI +Config carries, and retrieving skill content through an injectable store seam. + +The three layers of the feature sit in three modules, and the dependencies run +one way only: + +- ``skills_core.py`` — the store and telemetry seams, module state, integrity + verification, and store resolution. Shared, and imports neither of the others. +- ``skills.py`` (this file) — ``skill_refs``, the accessors, and + ``InMemorySkillStore``. +- ``skills_fs.py`` — the highest-blast-radius layer, the one that writes to a + customer's disk. It owns the manifest format and the on-disk filenames; + nothing here knows about the filesystem. + +``_set_store``, ``_set_emitter_for_testing`` and ``_clear_state`` live here +because this module is the documented injection path; the +state they mutate lives in ``skills_core``. """ from __future__ import annotations import logging +from collections.abc import Callable, Sequence +from typing import Any -from .types import AiConfigRep, SkillReference +from . import skills_core +from .skills_core import ( + SKILL_OBJECT_KIND, + list_raw_objects, + log_withholding_summary, + newest_by_key, + reference_target, + require_store, + resolve_from_store, + verify_raw_skill, +) +from .types import AiConfigRep, Skill, SkillReference from .types_validation import ( is_valid_skill_key, is_valid_skill_version, @@ -19,16 +46,140 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Injection points +# --------------------------------------------------------------------------- +# +# These three names are the documented seam: ``init_client`` and ``shutdown`` +# call them, and tests inject through them. They delegate to ``skills_core``, +# which owns the state, so that there is exactly one store and one emitter no +# matter which layer reaches for them. + + +# Bound directly to the implementations rather than wrapped: a one-line +# delegation per name would give every state mutation two definitions and two +# docstrings to keep in agreement, which is the drift these names exist to +# avoid. ``_set_emitter_for_testing`` keeps its distinct name because it has no +# production caller. +_set_store = skills_core.set_store +_set_emitter_for_testing = skills_core.set_emitter +_clear_state = skills_core.clear_state + + +class InMemorySkillStore: + """ + A skill store backed by plain dicts. + + Ships for local development, tests, and bring-your-own-content injection. + Holds raw wire objects verbatim and performs no validation of its own — + verification belongs at the accessor boundary, where it applies to every + store equally. + + Several versions of one key coexist here, because they coexist in a real + delivery payload: the newest version of every skill, plus every version a + variation currently pins. ``get_object`` therefore selects on + ``(key, version)``, and ``version=None`` means "the newest held". + + An object whose ``version`` is not an integer >= 1 is still accepted and + still served, under its key alone. Withholding it is verification's job, not + the store's: a store that quietly refused it would make a malformed object + indistinguishable from an absent one, and no integrity signal would be + recorded. + """ + + def __init__(self, objects: dict[str, dict[str, Any]] | None = None) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + for object_key, raw in (objects or {}).items(): + self._place(object_key, raw) + + def _place(self, fallback_key: str, raw: dict[str, Any]) -> None: + """Files one raw object under its own identity, verbatim.""" + key = raw.get("key") if isinstance(raw, dict) else None + if not isinstance(key, str): + key = fallback_key + version = raw.get("version") if isinstance(raw, dict) else None + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def put(self, raw: dict[str, Any]) -> None: + """ + Adds or replaces a raw skill object, keyed by its own ``key`` and + ``version`` fields. + + Putting a second version of a key keeps both; putting the same + ``(key, version)`` twice replaces it. + + Notifies every skill-kind listener with the raw object as a single + positional argument. No validation happens here — verification belongs at + the accessor boundary, where it applies to every store equally — so a + listener sees exactly what was put, unverified. + """ + key = raw.get("key") + if not isinstance(key, str): + raise ValueError("a raw skill object must carry a string 'key'") + self._place(key, raw) + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener(raw) + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry when the pin does not match + # anything well-formed, so a malformed object reaches verification and + # is withheld with a signal rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + """ + Every object held, one entry per ``(key, version)``. + + The dict keys are opaque store-internal identifiers, as ``SkillStore`` + documents. Do not parse them and do not assume one entry per skill key. + """ + if kind != SKILL_OBJECT_KIND: + return {} + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called with each raw object ``put`` under *kind*. + + Only ``kind == SKILL_OBJECT_KIND`` is ever notified, because ``put`` only + accepts skill objects; a listener registered under any other kind is + recorded and never fires. + """ + self._listeners.setdefault(kind, []).append(fn) + # --------------------------------------------------------------------------- # Reference discovery # --------------------------------------------------------------------------- + + def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: """ Projects a resolved AI Config's ``skills`` array into typed references. A pure projection — no network, no client, no store, no telemetry. Returns - ``[]`` when the config carries no skills. + ``[]`` when the config carries no skills. Compose it with the accessors for + per-context resolution: ``await get_skills(skill_refs(config))``. A config that came through ``parse_ai_config`` never contains an invalid entry — parsing fails closed on one. A hand-built dict can, and a silently @@ -70,3 +221,82 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: else: refs.append(SkillReference(key=key, version=version)) return refs + + +# --------------------------------------------------------------------------- +# Content accessors +# --------------------------------------------------------------------------- + + +async def get_skill(key: str, *, version: int | None = None) -> Skill | None: + """ + Retrieves one verified skill by key. + + ``version=None`` means the newest version the store holds; a specific + ``version`` asks the store for that version and returns it only when the + store answers with it. A payload holding several versions of one key + resolves a pin to the pinned version, not to the newest. + Returns ``None`` — never raises — when the skill is missing, the requested + version is not the one held, or verification fails. Raises ``RuntimeError`` + only when no skill store is configured. + + There is no context parameter: skills have no targeting, so the SDK + credentials fully determine availability. Compose per-context resolution + explicitly with ``get_skills(skill_refs(config))``. + """ + return resolve_from_store(require_store(), key, version).skill + + +async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: + """ + Retrieves a batch of verified skills. + + Accepts a mixed sequence of ``SkillReference`` values and bare key strings, + where a string means "the latest version". Results follow input order for + the skills that were found; entries that are missing, are the wrong version, + or fail verification are omitted rather than returned as placeholders — and + a run that omitted anything logs a count at WARN, so a batch that resolved + nothing is not silent. + """ + if isinstance(refs, str): + # str satisfies Sequence[str], so this type-checks; iterating it would + # silently look up one skill per character. + raise TypeError( + "get_skills takes a sequence of references; pass [key] rather than a " + f"bare string. Got {refs!r}." + ) + + store = require_store() + + requests = list(refs) + skills: list[Skill] = [] + for ref in requests: + key, wanted = reference_target(ref) + skill = resolve_from_store(store, key, wanted).skill + if skill is not None: + skills.append(skill) + log_withholding_summary("requested skills", len(requests), len(skills)) + return skills + + +async def all_skills() -> list[Skill]: + """ + Retrieves every verified skill the store currently holds. + + Skills that fail verification are omitted. Raises ``RuntimeError`` only when + no skill store is configured. + """ + objects, error = list_raw_objects(require_store()) + if error is not None: + return [] + + # One entry per key at its newest version: ``all_objects`` may hold several + # versions of one key, and a list carrying two of them is not a set of skills. + candidates = newest_by_key(objects) + skills: list[Skill] = [] + for _object_key, raw in candidates: + skill = verify_raw_skill(raw) + if skill is not None: + skills.append(skill) + log_withholding_summary("skills held by the store", len(candidates), len(skills)) + return skills diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py new file mode 100644 index 00000000..c478be93 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -0,0 +1,628 @@ +""" +Agent Skills — the internals ``skills`` and ``skills_fs`` both need. + +Extracted so the two layers above it share one implementation through an +explicit surface instead of reaching into each other's privates. Everything +here is package-internal — nothing in this module is exported from +``launchdarkly_ai_server`` except the two constants that are public API — and +the dependency runs one way: this module imports neither ``skills`` nor +``skills_fs``. + +What lives here, and why it has to be one copy: + +- **The store seam and the configured store.** One place holds the store, so + the accessors and the materialization path cannot disagree about whether one + is configured. +- **The telemetry seam.** Every signal the feature can emit is constructed by a + ``record_*`` function in this file and nowhere else, which is what makes the + three-signal allowlist enforceable by reading one section. ``emit`` is never + called from outside this module. +- **Integrity verification.** ``verified_bytes`` runs twice per skill by design + — once at the accessor boundary, and again immediately before a write, since a + ``Skill`` can also be constructed directly by a caller. Sharing the + implementation is what keeps the two passes from drifting — the signal's + property keys must match whichever layer caught the defect. +- **Store resolution.** ``resolve_from_store`` is the fetch-and-verify sequence + the accessors and the reconcile share, so its call sites cannot drift apart — + in particular on how a raising store is handled. + +Everything the store hands back is untrusted input; the transport is not part of +the trust boundary. Key, version, size, and content hash are revalidated here on +every pass. + +The store and emitter are injected through ``skills._set_store`` and +``skills._set_emitter_for_testing`` — those names are the documented seam, +and they delegate here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass +from typing import Any, Protocol + +from .types import Skill, SkillReference +from .types_validation import is_valid_skill_key, is_valid_skill_version + +logger = logging.getLogger(__name__) + +SKILL_OBJECT_KIND = "skill" +""" +The kind this SDK asks a store for. + +An **internal seam value**, deliberately not exported from the package root. It +is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` +and ``SkillStore.all_objects``, and a store adapter is free to map it onto +whatever the transport underneath actually uses — a delivery payload may well +carry skills under a broader kind with a narrower category, in which case +translating that pair to this one value is the adapter's job. + +Exporting it would publish an SDK-side seam string as though it were the wire +contract, which is a claim this side cannot make and would be hard to walk back +once a caller depends on it. A store that needs to agree on a kind agrees with +whatever the SDK hands it, which is this constant reached through +``launchdarkly_ai_server.skills_core``. +""" + +MAX_SKILL_CONTENT_BYTES = 64 * 1024 +""" +Hard cap on skill content. Legitimately delivered skills are well under this +bound, so anything larger is withheld regardless of whether its hash checks out. + +Deliberately **not** exported from the package root, unlike the on-disk and +on-the-wire constants beside it. Those are values this SDK defines and a caller +may need to agree with; this one is a local enforcement bound on content the +platform produces, so publishing it would semver-lock a number this side does +not own — and a caller pre-flighting "will my skill fit?" against it would be +reading the client's guess rather than the real limit. The reason string from +``verified_bytes`` already reports the bound when it is what withheld content. +""" + +_LANGUAGE = "python" + +_SHA256_HEX = re.compile(r"\A[0-9a-f]{64}\Z") +"""What a legitimate content hash looks like. Anything else is redacted before +it reaches telemetry — ``contentHash`` is attacker-controlled, and a store that +put the skill body there would otherwise leak it into a signal.""" + +_SIGNAL_INTEGRITY_FAILURE = "AgentControl Skill Integrity Failure" +_SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" +_SIGNAL_REVOKED = "AgentControl Skill Revoked Received" + +NO_STORE_MESSAGE = ( + "No skill store is configured, so skill content cannot be retrieved. Configure " + 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' + "available for local development and testing." +) + + +# --------------------------------------------------------------------------- +# The store seam +# --------------------------------------------------------------------------- + + +class SkillStore(Protocol): + """ + Structural interface every source of skill content satisfies. + + Duck-typed on purpose, mirroring how the LaunchDarkly client interface works + in this package: pass any object carrying these methods. + + ``add_listener(kind, fn)`` is part of the seam but + **optional**, which is why it is deliberately not declared here: a Protocol + member is required for structural compatibility, so declaring it would reject + every store that does not implement it. Nothing in this module calls it — it + exists for the delivery transport to push updates through. + + The raw objects a store serves are wire-shaped, with camelCase field names + identical across language implementations:: + + {"key": "pdf-extraction", "version": 2, "content": "---\\n...", + "contentHash": "9f3a...", "name": "PDF Extraction", "description": "..."} + + **Version is part of the lookup identity, not a filter applied afterwards.** + A delivery payload holds the newest version of every skill *and* every + version any variation currently pins, so two versions of one key coexist + routinely. A seam keyed by key alone cannot express "the one this variation + pinned": it would answer with the newest and the caller would then have to + reject it, which turns a pinned reference into a missing skill. So + ``get_object`` takes the wanted version, and ``version=None`` means "the + newest you hold". + + ``all_objects`` returns one entry per *(key, version)* the store holds. Its + dict keys are **opaque store-internal identifiers** — do not parse them, and + do not assume one entry per skill key. Identity is read off each object's own + ``key`` and ``version`` fields, which are revalidated here anyway because + everything a store serves is untrusted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: ... + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: ... + + +# --------------------------------------------------------------------------- +# Telemetry seam +# --------------------------------------------------------------------------- + + +class _TelemetryEmitter(Protocol): + def record(self, signal: str, properties: dict[str, Any]) -> None: ... + + +class _NoOpEmitter: + """ + The default emitter. + + No skills telemetry leaves the process in this release: ``client.track()`` is + the wrong channel (it needs an LD context, spends the customer's event + volume, and lands in their data export), and the diagnostic-event channel + that would be right has no wrapper-SDK extension point yet. Signals are + recorded through this seam so the eventual transport drops in behind it + without touching a single call site. + """ + + def record(self, signal: str, properties: dict[str, Any]) -> None: + return None + + +_NOOP_EMITTER: _TelemetryEmitter = _NoOpEmitter() + +# --------------------------------------------------------------------------- +# Module state +# --------------------------------------------------------------------------- + +_store: SkillStore | None = None +_emitter: _TelemetryEmitter = _NOOP_EMITTER +"""Never ``None``: "no emitter installed" is spelled as the no-op, so ``emit`` +has one code path instead of re-deciding on every signal.""" + + +def set_store(store: Any) -> None: + """ + Replaces the configured store. + + Reached through ``skills._set_store``, which is the documented seam; see that + function for who calls it and why it has no test-only twin. + """ + global _store + _store = store + + +def set_emitter(emitter: Any) -> None: + """Replaces the telemetry emitter. Reached through + ``skills._set_emitter_for_testing``.""" + global _emitter + _emitter = emitter + + +def clear_state() -> None: + """Drops both the store and the emitter. Reached through ``skills._clear_state``.""" + global _store, _emitter + _store = None + _emitter = _NOOP_EMITTER + + +def get_store() -> SkillStore | None: + """The configured store, or ``None``. The only reader of the global.""" + return _store + + +def require_store() -> SkillStore: + store = get_store() + if store is None: + raise RuntimeError(NO_STORE_MESSAGE) + return store + + +def emit(signal: str, properties: dict[str, Any]) -> None: + """ + Records one signal. Never raises into the calling operation — a broken + emitter must not be able to fail a retrieval or a reconcile. + """ + try: + _emitter.record(signal, properties) + except Exception: + logger.warning("Skills telemetry emitter raised; ignoring", exc_info=True) + + +def record_integrity_failure( + skill_key: str, + reason: str, + *, + version: Any = None, + expected_hash: Any = None, + observed_hash: str | None = None, +) -> None: + """ + Records an integrity failure. Carries hashes and byte counts only — the skill + body never appears in a signal, a log line, or an error message. + """ + # Both of these come off the wire, so neither may be echoed verbatim: a store + # that set contentHash (or key) to the skill body would otherwise publish the + # body itself. Shape-check, then redact. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} + if is_valid_skill_version(version): + properties["version"] = version + if isinstance(expected_hash, str): + properties["expected_hash"] = ( + expected_hash + if _SHA256_HEX.match(expected_hash) + else "" + ) + if observed_hash is not None: + properties["observed_hash"] = observed_hash + + logger.error("Skill '%s' failed integrity verification: %s", safe_key, reason) + emit(_SIGNAL_INTEGRITY_FAILURE, properties) + + +def record_materialized( + skill_key: str, content_bytes: int, content_hash: str, reconcile_action: str +) -> None: + """ + Records a materialization. Deliberately carries no ``target_path`` and no + filesystem path of any kind — the same reasoning that keeps the skill body + out of telemetry keeps the customer's directory layout out. Paths live in the + returned ``ReconcileReport``, which is user-facing API rather than telemetry. + """ + emit( + _SIGNAL_MATERIALIZED, + { + "skill_key": skill_key, + "content_bytes": content_bytes, + "content_hash": content_hash, + "reconcile_action": reconcile_action, + "language": _LANGUAGE, + }, + ) + + +def record_revoked(skill_key: str, version: Any) -> None: + """ + Records a revocation — a prune that removed a formerly managed skill. + + Lives here with the other two recorders rather than at the prune site so the + signal allowlist is maintained in one place: every signal this SDK can emit + is visible in this section of this module, and nothing outside it touches + ``emit``. + """ + # Both fields come off the manifest, which is untrusted — same rule as + # ``record_integrity_failure``: shape-check, then redact, so a hand-edited + # manifest cannot plant an arbitrary string in a signal. + safe_key = skill_key if is_valid_skill_key(skill_key) else "" + properties: dict[str, Any] = { + "skill_key": safe_key, + "removed_from_disk": True, + "language": _LANGUAGE, + } + if is_valid_skill_version(version): + properties["version"] = version + emit(_SIGNAL_REVOKED, properties) + + +# --------------------------------------------------------------------------- +# Integrity verification +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class VerifiedContent: + """Content that passed integrity verification.""" + + encoded: bytes + """The verbatim bytes, exactly as hashed.""" + content_hash: str + """The locally computed sha256 — never the caller's expected value.""" + + +@dataclass(frozen=True) +class VerificationFailure: + """Why content did not pass. The reason is safe to show a caller.""" + + reason: str + + +def verified_bytes( + key: str, content: str | bytes, expected_hash: str, version: int +) -> VerifiedContent | VerificationFailure: + """ + The whole content half of integrity verification: encode, size, hash. + + Accepts either shape content legitimately arrives in. Wire-shaped ``str`` + input — a raw store object's JSON string — is UTF-8 encoded here, once, and + this is the only place that encode happens. ``bytes`` input is an already + verified ``Skill.content`` being re-verified, and is hashed directly: those + bytes are the verbatim value, so re-encoding does not apply. + + Returns the verbatim bytes and their locally computed sha256, or a + human-readable reason — having already recorded the integrity signal, so the + signal's property set cannot depend on which caller noticed. The hash handed + back is the one computed here, never the caller's expected value: the two are + equal on this path by construction, and returning the locally derived one + keeps an attacker-supplied string out of ``Skill``. + + The two outcomes are distinct types rather than a ``tuple | str`` union so a + call site reads as "verification failed" instead of "the result is a string", + and so a future success payload carrying a ``str`` cannot silently invert the + discrimination. + + This runs twice per skill by design: once at the accessor boundary, and again + immediately before a write, because a ``Skill`` can also be constructed + directly by a caller. Sharing the implementation is what keeps those two + passes from drifting — the property keys must match. + + The second pass re-hashes bytes the first pass already hashed. That + redundancy is deliberate: it is negligible next to the write it guards, and + carrying the first pass's verdict forward would put a "trust the value + computed upstream" branch inside the one function whose entire job is not to. + """ + if isinstance(content, bytes): + encoded = content + else: + try: + encoded = content.encode("utf-8") + except UnicodeEncodeError: + # json.loads turns a "\ud800" escape into an unpaired surrogate, which + # has no UTF-8 encoding. There are no bytes the server could have + # hashed, so this is not authentic content. Never use + # errors="surrogatepass" here: that would fabricate bytes and could + # satisfy the hash comparison. + reason = "content is not encodable as UTF-8" + record_integrity_failure( + key, reason, version=version, expected_hash=expected_hash + ) + return VerificationFailure(reason) + + if len(encoded) > MAX_SKILL_CONTENT_BYTES: + reason = ( + f"content is {len(encoded)} bytes, over the " + f"{MAX_SKILL_CONTENT_BYTES} byte cap" + ) + record_integrity_failure( + key, reason, version=version, expected_hash=expected_hash + ) + return VerificationFailure(reason) + + # sha256, lowercase hex, over the verbatim bytes — no canonicalization and + # no content parsing of any kind anywhere in the integrity path. + observed_hash = hashlib.sha256(encoded).hexdigest() + if observed_hash != expected_hash: + record_integrity_failure( + key, + "content hash mismatch", + version=version, + expected_hash=expected_hash, + observed_hash=observed_hash, + ) + return VerificationFailure("content hash mismatch") + + return VerifiedContent(encoded=encoded, content_hash=observed_hash) + + +def verify_raw_skill(raw: Any) -> Skill | None: + """ + Turns one untrusted raw store object into a ``Skill``, or withholds it. + + On any failure the skill is treated as missing, the integrity signal is + recorded, and an error is logged. No unverified content is ever returned to + user code. + """ + if not isinstance(raw, dict): + record_integrity_failure("", "raw skill object is not an object") + return None + + key = raw.get("key") + if not is_valid_skill_key(key): + record_integrity_failure( + key if isinstance(key, str) else "", + "key is not a valid skill key", + ) + return None + + version = raw.get("version") + if not is_valid_skill_version(version): + record_integrity_failure(key, "version is not an integer >= 1") + return None + + content = raw.get("content") + if not isinstance(content, str): + record_integrity_failure( + key, "content is missing or not a string", version=version + ) + return None + + expected_hash = raw.get("contentHash") + if not isinstance(expected_hash, str): + record_integrity_failure( + key, "contentHash is missing or not a string", version=version + ) + return None + + verified = verified_bytes(key, content, expected_hash, version) + if isinstance(verified, VerificationFailure): + return None + + name = raw.get("name") + description = raw.get("description") + return Skill( + key=key, + version=version, + content=verified.encoded, + content_hash=verified.content_hash, + name=name if isinstance(name, str) else None, + description=description if isinstance(description, str) else None, + ) + + +def log_withholding_summary(subject: str, requested: int, resolved: int) -> None: + """ + One WARN per run when content was withheld, naming the counts. + + Every individual withholding already records an integrity signal and an error + log line, but a caller reading logs at WARN sees neither. That matters most in + the case where *nothing* verified — a payload built before ``contentHash`` is + populated, say — because the feature then returns an empty result that is + indistinguishable from "this project has no skills". A run-level summary is + the difference between a silent no-op and a visible one. + + Called once per batch retrieval, not once per skill, so a large withholding + run does not itself become the noise. + """ + withheld = requested - resolved + if withheld <= 0: + return + if resolved == 0: + logger.warning( + "All %d %s were withheld and no skill content is available. Every " + "object failed verification — check that the delivered objects carry " + "a contentHash matching the sha256 of their content.", + requested, + subject, + ) + return + logger.warning( + "%d of %d %s were withheld and are unavailable; see the preceding errors " + "for the per-skill reason.", + withheld, + requested, + subject, + ) + + +def store_raised(exc: Exception) -> str: + """The one wording for "the store could not answer", used by every path.""" + return f"the skill store raised {type(exc).__name__}: {exc}" + + +def list_raw_objects( + store: SkillStore, +) -> tuple[dict[str, dict[str, Any]], str | None]: + """ + Every raw object the store holds, or the reason it could not answer. + + One entry per *(key, version)*, under keys that are opaque to this SDK — see + ``SkillStore``. Callers that need one skill per key have to collapse the + result themselves; ``newest_by_key`` does it. + + Returns the reason rather than raising, because both callers need the + distinction between "no skills" and "the store is broken" — and they need it + worded identically. Letting the exception out instead would make each of + them re-derive the log line and the message, which is the drift this module + exists to prevent. + """ + try: + objects = store.all_objects(SKILL_OBJECT_KIND) + except Exception as exc: + logger.error("Skill store raised while listing skills", exc_info=True) + return {}, store_raised(exc) + return (objects if isinstance(objects, dict) else {}), None + + +def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: + """ + One raw object per skill key — the highest version of each, paired with the + store key it was served under. + + ``all_objects`` may hold several versions of one key, and both callers that + consume the whole store want one skill per key: ``all_skills`` because a list + holding two versions of one key is not a set of skills, and the ``"*"`` + reconcile because ``//SKILL.md`` is a single path and writing it + twice in one run is a bug rather than a policy. + + The store key is carried through rather than discarded because the reconcile + attributes a failure to it when the object's own key is unusable. + + Objects too malformed to carry a usable key and version are **kept**, not + dropped, so verification is what withholds them: a silently dropped object + falls out of the requested set, and prune would then delete the last + known-good copy already on disk. + """ + best: dict[str, tuple[str, Any]] = {} + unusable: list[tuple[str, Any]] = [] + for object_key, raw in objects.items(): + skill_key = raw.get("key") if isinstance(raw, dict) else None + version = raw.get("version") if isinstance(raw, dict) else None + if not is_valid_skill_key(skill_key) or not is_valid_skill_version(version): + unusable.append((object_key, raw)) + continue + held = best.get(skill_key) + if held is None or version > held[1]["version"]: + best[skill_key] = (object_key, raw) + return list(best.values()) + unusable + + +# --------------------------------------------------------------------------- +# Resolution internals — shared with the materialization path +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Resolution: + """One key resolved against a store: the skill, or why there is none.""" + + skill: Skill | None = None + error: str | None = None + unavailable: bool = False + """ + ``True`` when the *store* could not answer — it raised — rather than when it + answered "no". Only the former suppresses pruning: deleting managed files + because a lookup failed would turn an outage into data loss. + """ + + +def resolve_from_store( + store: SkillStore, key: str, wanted_version: int | None +) -> Resolution: + """ + Fetches one key and verifies it — the sequence the accessors and the + materialization path share. + + Written once on purpose, so the call sites cannot drift apart — in + particular on the policy for a raising store. + + ``wanted_version`` goes *into* the lookup, because a store may hold several + versions of one key and only it can pick between them; ``None`` asks for the + newest. The equality check afterwards is kept as a **defense**, not as the + selection mechanism: the store is untrusted, so an answer that is not the + version that was asked for is withheld rather than returned. + """ + try: + raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) + except Exception as exc: + logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) + return Resolution(error=store_raised(exc), unavailable=True) + + if not isinstance(raw, dict): + return Resolution( + error=f"skill '{key}' is not available from the configured skill store" + ) + + skill = verify_raw_skill(raw) + if skill is None: + return Resolution( + error=f"skill '{key}' failed integrity verification and was withheld" + ) + if wanted_version is not None and skill.version != wanted_version: + return Resolution( + error=( + f"skill '{key}' version {wanted_version} is not available " + f"(the store holds version {skill.version})" + ) + ) + return Resolution(skill=skill) + + +def reference_target(item: SkillReference | str) -> tuple[str, int | None]: + """Normalises a reference-or-key into ``(key, wanted version)``. + + A bare string means "the latest version the store holds". + """ + if isinstance(item, str): + return item, None + return item.key, item.version diff --git a/packages/client/tests/conftest.py b/packages/client/tests/conftest.py index 6c140200..17fbb68b 100644 --- a/packages/client/tests/conftest.py +++ b/packages/client/tests/conftest.py @@ -1,7 +1,14 @@ +import hashlib +from collections.abc import Iterator +from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import InMemorySkillStore + @pytest.fixture def mock_ld_client() -> MagicMock: @@ -36,3 +43,108 @@ def mock_tracer(mock_span: MagicMock) -> MagicMock: tracer.start_as_current_span.return_value.__exit__ = MagicMock(return_value=False) tracer.start_span.return_value = mock_span return tracer + + +# --------------------------------------------------------------------------- +# Agent Skills helpers +# +# Exposed as fixtures rather than importable module-level helpers: pytest runs +# with --import-mode=importlib and the tests directory is not a package, so +# sibling imports from conftest are not reliable. +# --------------------------------------------------------------------------- + +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + + +class _RecordingEmitter: + """Telemetry seam double — records (signal, properties) pairs.""" + + def __init__(self) -> None: + self.records: list[tuple[str, dict[str, Any]]] = [] + + def record(self, signal: str, properties: dict[str, Any]) -> None: + self.records.append((signal, properties)) + + def signals(self, name: str) -> list[dict[str, Any]]: + return [props for sig, props in self.records if sig == name] + + +class _ThrowingEmitter: + """Telemetry seam double whose record() always raises.""" + + def record(self, signal: str, properties: dict[str, Any]) -> None: + raise RuntimeError("emitter exploded") + + +@pytest.fixture +def make_raw_skill() -> Any: + """Factory for wire-shaped raw store objects with a correct contentHash.""" + + def _make( + key: str = "test-skill", + version: int = 1, + content: str = SKILL_BODY, + **overrides: Any, + ) -> dict[str, Any]: + obj: dict[str, Any] = { + "key": key, + "version": version, + "content": content, + "contentHash": hashlib.sha256(content.encode("utf-8")).hexdigest(), + "name": "Test Skill", + "description": "A skill used in tests.", + } + obj.update(overrides) + return obj + + return _make + + +@pytest.fixture +def store() -> InMemorySkillStore: + """An in-memory store, wired in as the configured store for the test.""" + s = InMemorySkillStore() + skills_module._set_store(s) + return s + + +class _ExplodingStore: + """Store double whose every read raises — the "transport is down" case.""" + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +@pytest.fixture +def exploding_store() -> _ExplodingStore: + """A raising store, wired in as the configured store for the test.""" + s = _ExplodingStore() + skills_module._set_store(s) + return s + + +@pytest.fixture +def reset_skill_state() -> Iterator[None]: + """Clears client, store, and emitter module state around one test. + + Opted into per module with ``pytestmark = pytest.mark.usefixtures(...)`` + rather than being autouse here: autouse would newly reset lifecycle state + for every test in every module in this directory, which is a behaviour change + well outside the skills tests. + """ + lifecycle_module._reset_for_testing() + yield + lifecycle_module._reset_for_testing() + + +@pytest.fixture +def recording_emitter() -> _RecordingEmitter: + return _RecordingEmitter() + + +@pytest.fixture +def throwing_emitter() -> _ThrowingEmitter: + return _ThrowingEmitter() diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 38ca34f2..e99a8957 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1,24 +1,57 @@ """ -Agent Skills — reference discovery: the types and the projection from a -resolved AI Config. +Tests for Agent Skills types, reference discovery, content accessors, +integrity verification, and the telemetry seam. """ from __future__ import annotations import dataclasses import hashlib +import json from typing import Any +from unittest.mock import MagicMock import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.skills as skills_module from launchdarkly_ai_server import ( + InMemorySkillStore, Skill, SkillReference, + all_skills, + get_client, + get_skill, + get_skills, + init_client, + shutdown, skill_refs, ) SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" + +# The three signal names are an allowlist, not a floor: any +# other name reaching the emitter is a regression. +APPROVED_SIGNALS = frozenset({INTEGRITY_SIGNAL, MATERIALIZED_SIGNAL, REVOKED_SIGNAL}) + +# These two were considered and deliberately excluded from SDK emission — +# named explicitly rather than relying on the subset check to be read as +# covering them. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + def _hash(content: str) -> str: """sha256, lowercase hex, over verbatim utf-8 bytes.""" @@ -40,6 +73,61 @@ def _skill( ) +# --------------------------------------------------------------------------- + +UNENCODABLE_BODIES = ( + json.loads(r'"hi \ud800 there"'), # lone high surrogate + # A lone *low* surrogate, which is the one range errors="surrogateescape" + # smuggles through (as a raw 0x80 byte) while raising on everything else. + json.loads(r'"hi \udc80 there"'), +) + +NON_STRICT_HANDLERS = ( + "surrogatepass", + "surrogateescape", + "replace", + "ignore", + "backslashreplace", + "xmlcharrefreplace", + "namereplace", +) +"""Every ``str.encode`` error handler that is not ``strict``. + +``verified_bytes`` must use none of them: each one *fabricates* bytes for input +that has no encoding, and fabricated bytes can satisfy the hash comparison. +""" + + +def _fabricated_hash_cases() -> list[Any]: + """One case per (body, handler) pair the handler can actually encode. + + Each carries the sha256 of the bytes *that* handler would have produced, so + the case is not vacuous: an implementation that reached for the handler + would encode successfully, match the pinned hash, and return content + LaunchDarkly never delivered. Handlers that raise on a given body are + skipped — for that input they are as strict as ``strict``, so there is + nothing to detect. + """ + cases: list[Any] = [] + for index, body in enumerate(UNENCODABLE_BODIES): + for handler in NON_STRICT_HANDLERS: + try: + fabricated = body.encode("utf-8", errors=handler) + except UnicodeEncodeError: + continue + cases.append( + pytest.param( + body, + hashlib.sha256(fabricated).hexdigest(), + id=f"body{index}-{handler}", + ) + ) + return cases + + +FABRICATED_HASH_CASES = _fabricated_hash_cases() + + class TestSkillTypes: """Immutability and optional metadata.""" @@ -106,6 +194,11 @@ def test_returns_typed_references_in_order(self) -> None: ] assert all(isinstance(r, SkillReference) for r in refs) + def test_emits_no_telemetry(self, recording_emitter: Any) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + skill_refs(self._config(skills=[{"key": "a", "version": 1}])) + assert recording_emitter.records == [] + def test_dropped_entries_are_logged(self, caplog: pytest.LogCaptureFixture) -> None: """A shortened projection is never silent. @@ -137,3 +230,925 @@ def test_requires_no_client_or_store(self, mock_ld_client: Any) -> None: refs = skill_refs(self._config(skills=[{"key": "a", "version": 2}])) assert refs == [SkillReference(key="a", version=2)] mock_ld_client.track.assert_not_called() + + +class TestPackageExports: + """ + What is and is not part of the public surface. + + The literal values are spelled out on purpose: this is the one place the + constants themselves are asserted, so importing them to build the + expectation would make the assertion circular. + """ + + def test_content_cap_is_not_public_api(self) -> None: + """The content cap stays internal to ``skills_core`` — see the + ``MAX_SKILL_CONTENT_BYTES`` docstring there for why it is not exported.""" + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.MAX_SKILL_CONTENT_BYTES == 65536 + assert "MAX_SKILL_CONTENT_BYTES" not in package.__all__ + assert not hasattr(package, "MAX_SKILL_CONTENT_BYTES") + + def test_object_kind_is_not_public_api(self) -> None: + """The kind is an SDK-side seam value, not the wire contract. + + A store adapter maps whatever the transport calls a skill onto the value + this SDK passes it, so publishing the string would advertise a contract + this side does not own — and one that would be hard to walk back once a + caller depended on it. It stays reachable through ``skills_core`` for the + adapter that needs to agree with it. + """ + import launchdarkly_ai_server as package + from launchdarkly_ai_server import skills_core + + assert skills_core.SKILL_OBJECT_KIND == "skill" + assert "SKILL_OBJECT_KIND" not in package.__all__ + assert not hasattr(package, "SKILL_OBJECT_KIND") + + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "skill_refs", + "get_skill", + "get_skills", + "all_skills", + "SkillStore", + "InMemorySkillStore", + "Skill", + "SkillReference", + } + assert expected <= set(package.__all__) + + +class TestInMemorySkillStore: + """The public in-memory store implementation.""" + + def test_get_object_round_trips(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="pdf-extraction", version=2) + s = InMemorySkillStore({"pdf-extraction": raw}) + assert s.get_object("skill", "pdf-extraction") == raw + + def test_get_object_unknown_key_returns_none(self) -> None: + assert InMemorySkillStore().get_object("skill", "nope") is None + + def test_put_then_get(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + raw = make_raw_skill(key="a") + s.put(raw) + assert s.get_object("skill", "a") == raw + + def test_all_objects_returns_everything(self, make_raw_skill: Any) -> None: + """Asserted on the object bodies, not the dict keys. + + ``all_objects`` keys are opaque store-internal identifiers — the seam + documents them as such — so a test that pinned their spelling would be + asserting an implementation detail the contract disclaims. + """ + s = InMemorySkillStore() + first = make_raw_skill(key="a") + second = make_raw_skill(key="b") + s.put(first) + s.put(second) + held = s.all_objects("skill").values() + assert len(held) == 2 + assert first in held + assert second in held + + def test_all_objects_holds_every_version_of_one_key( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=2, content="two\n") + s.put(v1) + s.put(v2) + held = list(s.all_objects("skill").values()) + assert len(held) == 2 + assert v1 in held + assert v2 in held + + def test_put_replaces_only_the_same_key_and_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="first\n")) + replacement = make_raw_skill(key="a", version=1, content="second\n") + s.put(replacement) + assert list(s.all_objects("skill").values()) == [replacement] + + def test_get_object_with_a_version_selects_that_version( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + v1 = make_raw_skill(key="a", version=1, content="one\n") + v2 = make_raw_skill(key="a", version=3, content="three\n") + s.put(v1) + s.put(v2) + assert s.get_object("skill", "a", 1) == v1 + assert s.get_object("skill", "a", 3) == v2 + + def test_get_object_without_a_version_selects_the_newest( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + newest = make_raw_skill(key="a", version=7, content="seven\n") + s.put(newest) + s.put(make_raw_skill(key="a", version=4, content="four\n")) + assert s.get_object("skill", "a") == newest + + def test_get_object_unknown_version_falls_back_to_a_malformed_object( + self, make_raw_skill: Any + ) -> None: + """A malformed object must reach verification, not read as absent. + + An object whose ``version`` is unusable is filed under its key alone. A + pinned lookup that finds nothing well-formed serves it anyway, so + verification withholds it and records an integrity signal — a store that + returned ``None`` here would make tampering indistinguishable from a + skill that was never delivered. + """ + s = InMemorySkillStore() + malformed = make_raw_skill(key="a", version="two") + s.put(malformed) + assert s.get_object("skill", "a", 2) == malformed + assert s.get_object("skill", "a") == malformed + + def test_all_objects_unknown_kind_is_empty(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + s.put(make_raw_skill(key="a")) + assert s.all_objects("flag") == {} + + def test_put_notifies_skill_kind_listeners(self, make_raw_skill: Any) -> None: + """``add_listener`` is part of the seam, so its one + implementation carries a smoke test for the callback contract: the raw + object, verbatim and unverified, as a single positional argument.""" + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + raw = make_raw_skill(key="a") + + s.put(raw) + + assert seen == [raw] + + def test_put_does_not_notify_other_kind_listeners( + self, make_raw_skill: Any + ) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("flag", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + +class TestStoreConfiguration: + """Store wiring on the lifecycle layer.""" + + async def test_configured_via_init_client_option( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + skill = await get_skill("a") + assert skill is not None + assert skill.key == "a" + + async def test_get_skill_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_get_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await get_skills([SkillReference(key="a", version=1)]) + + async def test_all_skills_raises_actionably_when_no_store(self) -> None: + with pytest.raises(RuntimeError, match="skill store"): + await all_skills() + + async def test_shutdown_clears_the_store( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=mock_ld_client) + assert await get_skill("a") is not None + + await shutdown() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_skill_store_is_applied_on_every_init_client_call( + self, make_raw_skill: Any + ) -> None: + """``skillStore`` is the one option a second call applies. + + ``init_client`` is idempotent for the client singleton, and on a second + call every other option is ignored. ``skillStore`` is applied anyway, + on purpose: it is what lets a client that was lazily auto-initialized, + or initialized without a store, be given one afterwards. Both halves are + asserted on the same pair of calls, because each is meaningless without + the other. + """ + first_store = InMemorySkillStore() + first_store.put(make_raw_skill(key="first")) + second_store = InMemorySkillStore() + second_store.put(make_raw_skill(key="second")) + + first_client = MagicMock() + second_client = MagicMock() + + await init_client(options={"skillStore": first_store}, client=first_client) + await init_client(options={"skillStore": second_store}, client=second_client) + + # Half one: the client singleton is unchanged — the second call is a + # no-op for it, so the second client was discarded. + assert get_client() is first_client + + # Half two: the store was nevertheless swapped. + assert await get_skill("second") is not None + assert await get_skill("first") is None + + async def test_init_client_without_a_store_leaves_the_configured_one( + self, make_raw_skill: Any + ) -> None: + """Only a non-None ``skillStore`` replaces the configured store. + + Otherwise a bare ``init_client()`` from an unrelated code path — the + lazy auto-init, say — would silently unconfigure skills. + """ + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + await init_client(options={"skillStore": store}, client=MagicMock()) + + await init_client(client=MagicMock()) + + assert await get_skill("a") is not None + + async def test_failed_init_client_leaves_no_store_configured( + self, monkeypatch: pytest.MonkeyPatch, make_raw_skill: Any + ) -> None: + """A raising ``init_client`` must not leave global state behind. + + Installing the store before the SDK-key check would leave the accessors + working against a store the application believes was never installed, + masking a failed initialization. + """ + monkeypatch.delenv("LD_SDK_KEY", raising=False) + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + + with pytest.raises(RuntimeError, match="No LaunchDarkly SDK key"): + await init_client(options={"skillStore": store}) + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + async def test_reset_for_testing_clears_the_store( + self, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + skills_module._set_store(store) + assert await get_skill("a") is not None + + lifecycle_module._reset_for_testing() + + with pytest.raises(RuntimeError, match="skill store"): + await get_skill("a") + + +class TestAccessorArgumentErrors: + """A bare string is a type error, not a reference. + + ``str`` satisfies ``Sequence[str]``, so the annotation on ``get_skills`` + admits a bare string and only this runtime guard catches it; iterating one + would look up a skill per character. Deliberately a *different* class from + ``write_skills``'s bare-string rejection, which is a ``ValueError`` because + a string is an accepted argument type there. + """ + + async def test_bare_string_raises_type_error( + self, store: InMemorySkillStore + ) -> None: + with pytest.raises(TypeError) as excinfo: + await get_skills("pdf-extraction") # type: ignore[arg-type] + + # The message has to name the fix, not merely reject the input. + assert "[key]" in str(excinfo.value) + + async def test_bare_string_is_rejected_before_the_store_is_consulted( + self, make_raw_skill: Any + ) -> None: + """The guard is an argument check, so it precedes store resolution. + + Asserting the raise alone would also pass if the string were iterated + into single-character lookups that all missed, so pin that no lookup + happened at all. + """ + looked_up: list[str] = [] + + class _RecordingStore: + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + looked_up.append(key) + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + skills_module._set_store(_RecordingStore()) + + with pytest.raises(TypeError): + await get_skills("abc") # type: ignore[arg-type] + + assert looked_up == [] + + +class TestGetSkill: + """Single-skill accessor.""" + + async def test_returns_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 2 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "Test Skill" + assert skill.description == "A skill used in tests." + + async def test_version_omitted_returns_newest_available( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=7)) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 7 + + async def test_exact_version_match_returns_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + skill = await get_skill("a", version=3) + assert skill is not None + assert skill.version == 3 + + async def test_other_version_returns_none( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + assert await get_skill("a", version=2) is None + assert await get_skill("a", version=4) is None + + async def test_missing_key_returns_none(self, store: InMemorySkillStore) -> None: + assert await get_skill("nope") is None + + async def test_multibyte_content_verifies( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + content = "---\nname: emoji\n---\n🚀 unicode ✅ body\n" + store.put(make_raw_skill(key="a", content=content)) + skill = await get_skill("a") + assert skill is not None + assert skill.content == content.encode("utf-8") + + +class TestGetSkills: + """Batch accessor.""" + + async def test_mixed_refs_and_strings( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="b", version=5)) + result = await get_skills([SkillReference(key="a", version=1), "b"]) + assert [s.key for s in result] == ["a", "b"] + + async def test_preserves_input_order( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await get_skills(["c", "a", "b"]) + assert [s.key for s in result] == ["c", "a", "b"] + + async def test_missing_entries_are_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + result = await get_skills(["a", "missing"]) + assert [s.key for s in result] == ["a"] + + async def test_version_mismatch_is_omitted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=2)) + result = await get_skills([SkillReference(key="a", version=1)]) + assert result == [] + + async def test_empty_input_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await get_skills([]) == [] + + async def test_integrity_failure_omitted_and_signalled( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good-a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good-b")) + + result = await get_skills(["good-a", "bad", "good-b"]) + + assert [s.key for s in result] == ["good-a", "good-b"] + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 1 + assert failures[0]["skill_key"] == "bad" + + +class TestAllSkills: + """All_skills accessor.""" + + async def test_returns_every_verified_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + result = await all_skills() + assert {s.key for s in result} == {"a", "b", "c"} + + async def test_empty_store_returns_empty_list( + self, store: InMemorySkillStore + ) -> None: + assert await all_skills() == [] + + async def test_omits_skills_that_fail_verification( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="bad", contentHash="deadbeef")) + result = await all_skills() + assert {s.key for s in result} == {"good"} + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + +class TestVersionPinning: + """ + A payload holds several versions of one key, and a pin has to resolve to the + version it names. + + Delivery serves the newest version of every skill *plus* every version any + variation currently pins, so this is the ordinary case rather than an edge + one. A seam keyed by key alone cannot express it: it answers with the newest + and the pin then reads as a missing skill. + """ + + async def _two_versions(self, store: Any, make_raw_skill: Any) -> None: + store.put(make_raw_skill(key="a", version=1, content="version one\n")) + store.put(make_raw_skill(key="a", version=2, content="version two\n")) + + async def test_pinned_old_version_resolves( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a", version=1) + assert skill is not None + assert skill.version == 1 + assert skill.content == b"version one\n" + + async def test_latest_resolves_alongside_the_pinned_old_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + skill = await get_skill("a") + assert skill is not None + assert skill.version == 2 + + async def test_both_lookups_succeed_against_one_store( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + pinned = await get_skill("a", version=1) + latest = await get_skill("a", version=2) + assert pinned is not None and pinned.version == 1 + assert latest is not None and latest.version == 2 + + async def test_get_skills_resolves_a_mix_of_pins( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await get_skills( + [ + SkillReference(key="a", version=1), + SkillReference(key="b", version=5), + "a", + ] + ) + assert [(s.key, s.version) for s in skills] == [("a", 1), ("b", 5), ("a", 2)] + + async def test_pin_to_a_version_the_store_does_not_hold_returns_none( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + assert await get_skill("a", version=9) is None + + async def test_all_skills_returns_one_entry_per_key_at_the_newest_version( + self, store: Any, make_raw_skill: Any + ) -> None: + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="b", version=5, content="bee\n")) + skills = await all_skills() + assert sorted((s.key, s.version) for s in skills) == [("a", 2), ("b", 5)] + + async def test_a_store_answering_with_the_wrong_version_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The post-fetch check is a defense, not the selection mechanism. + + The store is untrusted, so an answer that is not the version that was + asked for is withheld rather than returned. + """ + + class _WrongVersionStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key=key, version=99) + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_WrongVersionStore()) + assert await get_skill("a", version=1) is None + + +class TestWithholdingSummary: + """ + A run that withheld content says so at WARN, once. + + Every individual withholding already records an integrity signal and an + error line, but a caller reading logs at WARN sees neither — and the case + that matters most is a payload where *nothing* verifies, because the feature + then returns an empty result indistinguishable from "this project has no + skills". + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_total_withholding_warns_and_names_the_hash( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill)) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await all_skills() == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "contentHash" in warnings[0].getMessage() + + async def test_partial_withholding_warns_with_the_counts( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="good")) + store.put(self._tampered(make_raw_skill, key="bad")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + skills = await all_skills() + assert [s.key for s in skills] == ["good"] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "1 of 2" in warnings[0].getMessage() + + async def test_get_skills_warns_once_per_batch( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(self._tampered(make_raw_skill, key="a")) + store.put(self._tampered(make_raw_skill, key="b")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await get_skills(["a", "b"]) == [] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + + async def test_a_fully_resolved_run_is_silent( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + store.put(make_raw_skill(key="a")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert len(await all_skills()) == 1 + assert len(await get_skills(["a"])) == 1 + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +class TestIntegrityVerification: + """Mandatory verification at the accessor boundary.""" + + async def test_hash_mismatch_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="a" * 64)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_tampered_content_withholds_skill( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + raw = make_raw_skill(key="a") + raw["content"] = raw["content"] + "x" # hash now stale by one byte + store.put(raw) + assert await get_skill("a") is None + + async def test_oversize_content_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (64 * 1024 + 1) + store.put(make_raw_skill(key="a", content=oversize)) + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_content_at_size_cap_is_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + at_cap = "x" * (64 * 1024) + store.put(make_raw_skill(key="a", content=at_cap)) + skill = await get_skill("a") + assert skill is not None + assert len(skill.content) == 64 * 1024 + + async def test_key_at_length_bound_from_store_accepted( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The accepting side of the <= 256 bound. + + ``write_skills`` cannot reach this bound (a key is one directory name and + NAME_MAX is 255), so config validation and this accessor-side + revalidation are the only two layers where 256 is observable at all. The + rejecting side is ``test_invalid_key_from_store_rejected["x" * 257]``. + """ + key = "a" * 256 + store.put(make_raw_skill(key=key)) + skill = await get_skill(key) + assert skill is not None + assert skill.key == key + + @pytest.mark.parametrize( + "bad_key", + [ + "Evil", + "-leading-dash", + ".hidden", + "has space", + "a/b", + "../escape", + "", + "x" * 257, + ], + ) + async def test_invalid_key_from_store_rejected( + self, make_raw_skill: Any, bad_key: str + ) -> None: + """A hostile store may serve any key — the accessor revalidates.""" + raw = make_raw_skill(key="placeholder") + raw["key"] = bad_key + skills_module._set_store(InMemorySkillStore({bad_key: raw})) + assert await get_skill(bad_key) is None + + @pytest.mark.parametrize("bad_version", [0, -1, 2.5, "2", None, True]) + async def test_invalid_version_from_store_rejected( + self, make_raw_skill: Any, bad_version: Any + ) -> None: + raw = make_raw_skill(key="a", version=bad_version) + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["content"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_missing_content_hash_rejected(self, make_raw_skill: Any) -> None: + raw = make_raw_skill(key="a") + del raw["contentHash"] + skills_module._set_store(InMemorySkillStore({"a": raw})) + assert await get_skill("a") is None + + async def test_uppercase_hash_rejected( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """Hashes are lowercase hex; a non-canonical hash is not authentic.""" + store.put(make_raw_skill(key="a", contentHash=_hash(SKILL_BODY).upper())) + assert await get_skill("a") is None + + @pytest.mark.parametrize(("body", "fabricated_hash"), FABRICATED_HASH_CASES) + async def test_unencodable_content_withheld( + self, recording_emitter: Any, body: str, fabricated_hash: str + ) -> None: + """Content with no UTF-8 encoding is withheld, and the signal recorded. + + ``str.encode`` raises on a lone surrogate, so ``verified_bytes`` has an + exception to catch and never sees bytes for this content at all. + + The parametrization is what makes that observable. The guard must never + pass ``errors="surrogatepass"``, or any other non-strict handler: each + of them fabricates bytes for input that has no encoding, and fabricated + bytes can satisfy the hash comparison. Every case here supplies the + sha256 of the bytes one such handler would have produced, so an + implementation that reached for one would verify this object + successfully and hand back content LaunchDarkly never sent. An + arbitrary wrong hash would not catch that — the mismatch check would + reject the input before the encoder guard was reached. + """ + with pytest.raises(UnicodeEncodeError): + body.encode("utf-8") # the premise: there is no encoding to hash + + skills_module._set_emitter_for_testing(recording_emitter) + skills_module._set_store( + InMemorySkillStore( + { + "a": { + "key": "a", + "version": 1, + "content": body, + "contentHash": fabricated_hash, + } + } + ) + ) + + assert await get_skill("a") is None + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_store_error_does_not_leak_content( + self, exploding_store: Any + ) -> None: + assert await get_skill("a") is None + assert await get_skills(["a"]) == [] + assert await all_skills() == [] + + +class TestTelemetrySeam: + """Internal emitter seam, no client.track, no context.""" + + async def test_default_emitter_is_noop( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", contentHash="0" * 64)) + assert await get_skill("a") is None # no emitter injected, no raise + + async def test_integrity_signal_properties( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", version=4, contentHash="b" * 64)) + + await get_skill("a") + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["version"] == 4 + assert props["expected_hash"] == "b" * 64 + assert props["observed_hash"] == _hash(SKILL_BODY) + assert props["language"] == "python" + + async def test_skill_body_never_appears_in_signals( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="a", contentHash="c" * 64)) + + await get_skill("a") + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + # ``skill_key`` and ``expected_hash`` are copied off the wire, so a hostile + # store can smuggle the body through either one and publish it in a signal + # that is otherwise body-free. The sweep above cannot see that: it serves a + # well-formed 64-character digest under a valid key, so neither replacement + # branch ever runs, and it passes even against an implementation that copies + # both fields verbatim. These two cases are what make the rule observable. + # Both assert the body's *absence* rather than the placeholder's exact + # spelling, which is not part of the contract. + + async def test_body_smuggled_through_content_hash_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + body = "UNIQUE-SECRET-BODY-VIA-HASH" + skills_module._set_store( + InMemorySkillStore( + {"a": {"key": "a", "version": 1, "content": body, "contentHash": body}} + ) + ) + + assert await get_skill("a") is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_body_smuggled_through_the_key_is_redacted( + self, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + # Uppercase and a path separator, so this is not a valid skill key and + # the invalid-key branch is the one that has to redact it. + body = "UNIQUE-SECRET-BODY-VIA-KEY/../x" + skills_module._set_store( + InMemorySkillStore( + {body: {"key": body, "version": 1, "content": "x", "contentHash": "y"}} + ) + ) + + assert await get_skill(body) is None + + signals = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(signals) == 1 + for value in signals[0].values(): + assert body not in str(value) + + async def test_no_ld_track_calls_from_accessors( + self, make_raw_skill: Any, mock_ld_client: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="a")) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + await init_client(options={"skillStore": store}, client=mock_ld_client) + + await get_skill("a") + await get_skill("bad") + await get_skills(["a"]) + await all_skills() + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_operation( + self, store: InMemorySkillStore, make_raw_skill: Any, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + store.put(make_raw_skill(key="bad", contentHash="0" * 64)) + store.put(make_raw_skill(key="good")) + + assert await get_skill("bad") is None + good = await get_skill("good") + assert good is not None + assert good.key == "good" + + async def test_accessors_record_no_signal_outside_the_approved_set( + self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any + ) -> None: + """The three names are an allowlist, not a floor. + + Asserted over the recorded strings, so nothing here mandates a + particular module-level constant. The write-side half of this sweep is + ``test_write_skills_records_no_signal_outside_the_approved_set`` in + test_skills_fs.py, where all four reconcile actions can be staged. + + Guards the most likely regression: an implementation that also emits + ``AgentControl Skill Content Retrieved`` from ``get_skill``, or + ``AgentControl Skill SDK Reference Returned`` from ``skill_refs``, + passes every other test in this class. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(make_raw_skill(key="good")) + store.put(make_raw_skill(key="tampered", contentHash="0" * 64)) + + assert await get_skill("good") is not None + assert await get_skill("tampered") is None + await get_skills(["good", "tampered"]) + await all_skills() + skill_refs({"skills": [{"key": "good", "version": 1}]}) + + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + # Positive control: a subset assertion is satisfied vacuously by an + # implementation that records nothing at all. + assert INTEGRITY_SIGNAL in recorded From 1afb690e0048e824849166cbe7f4ebe793fa2ddc Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 28 Aug 2026 16:11:51 -0400 Subject: [PATCH 03/43] feat(client): a documented log record for skill integrity failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An integrity failure now writes a structured, machine-parseable ERROR record on the SDK's own logger, designed to be ingested by a SIEM and alerted on. This is the detection path that works when telemetry is off, and the only one that exists at all in an instance with no telemetry destination — so it is a documented contract rather than a debugging aid. The LD-side counter is left exactly as designed: opt-out respecting, no-op by default, property set unchanged. `reason_code` lives in the log record only. - `ld.skills.integrity_failure` is the stable event name, and it appears in the message text rather than only in `extra`. Severity cannot discriminate — a raising store also logs ERROR from this module — and the stdlib's default formatter drops `extra`, so an `extra`-only record is invisible under a plain `logging.basicConfig()`. - The message is the event name plus compact key-sorted JSON, so the line is greppable, `jq`-able, and byte-identical across LaunchDarkly's AI SDKs for the same input. The same mapping is attached as `extra["ld_skills"]`. - `reason_code` is a closed vocabulary of eight tokens, one per `record_integrity_failure` call site, typed as a `Literal` so a typo at a call site is a type error. - The record spreads the signal's properties rather than rebuilding them, so the two cannot drift on which fields are redacted or omitted. Optional fields are omitted, never nulled. No new untrusted value, and no path. Documented for customers in the README and for contributors in agents.md, including the full vocabulary, so a ninth reason cannot land in one language only. --- packages/client/README.md | 51 ++++ packages/client/agents.md | 54 ++++ .../src/launchdarkly_ai_server/skills_core.py | 128 ++++++++- packages/client/tests/test_skills.py | 269 ++++++++++++++++++ 4 files changed, 491 insertions(+), 11 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index f16a1eb2..ae572b3b 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -279,6 +279,57 @@ revalidate, and its size is within 64 KiB. Anything that fails is withheld and t missing — no unverified content ever reaches your code. A retrieval that withheld anything logs a count at WARN, so a run that resolved nothing is not silent. +#### Detecting integrity failures + +Every withheld skill emits one structured **ERROR** log record on the SDK's own logger +(`launchdarkly_ai_server.skills_core`), designed to be ingested by a SIEM and alerted on. +It is emitted **regardless of how telemetry is configured** — it is not conditional on any +opt-in, and it is the detection path that works when nothing leaves your process. + +The message text is the stable event name followed by compact JSON, so it is greppable and +`jq`-able under any handler configuration, and the same mapping is attached as +`extra["ld_skills"]` for a structured handler: + +``` +ERROR ld.skills.integrity_failure {"action":"withheld","event":"ld.skills.integrity_failure","expected_hash":"0000…0000","language":"python","observed_hash":"5fc8…6ec0","reason":"content hash mismatch","reason_code":"hash_mismatch","skill_key":"pdf-extraction","version":2} +``` + +**`ld.skills.integrity_failure` is a stability commitment.** It is the string to match on, +it will not be renamed, and the JSON keys are sorted so the line is byte-identical across +LaunchDarkly's AI SDKs for the same input. + +| Field | Description | +|---|---| +| `event` | Always `ld.skills.integrity_failure`. | +| `action` | Always `withheld` — the content was not returned to your code. | +| `skill_key` | The skill key, or `` when the delivered key was itself malformed. | +| `version` | The delivered version. Omitted when it was not a valid version. | +| `expected_hash` | The delivered `contentHash`, or `` when it was not one. Omitted when none was delivered. | +| `observed_hash` | The sha256 the SDK computed. Omitted when the failure happened before anything was hashed. | +| `reason_code` | A stable token naming the failure mode — see below. | +| `reason` | Human-readable detail, including byte counts where relevant. | +| `language` | Always `python`. | + +Absent optional fields are **omitted entirely** rather than emitted as `null`, so a field +existence check is meaningful. The skill body, and any attacker-controllable string that +could carry it, never appears in the record; neither does any filesystem path. + +| `reason_code` | Meaning | +|---|---| +| `not_an_object` | The delivered object was not a JSON object. | +| `invalid_key` | The key did not match `^[a-z0-9][a-z0-9-]*$` or exceeded 256 characters. | +| `invalid_version` | The version was not an integer ≥ 1. | +| `missing_content` | `content` was absent or not a string. | +| `missing_content_hash` | `contentHash` was absent or not a string. | +| `not_utf8` | The content string had no UTF-8 encoding, so there are no bytes that could have been hashed. | +| `over_size_cap` | The content exceeded the SDK's local size cap. | +| `hash_mismatch` | The computed sha256 did not match the delivered `contentHash`. | + +**`hash_mismatch` is the one worth paging on.** The other seven describe a malformed or +truncated payload; a mismatch means content was delivered whose bytes are not the bytes +LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and +treat `expected_hash` / `observed_hash` as the evidence pair. + **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version diff --git a/packages/client/agents.md b/packages/client/agents.md index f284c425..b5733567 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -264,6 +264,59 @@ Exactly three signals exist, and the list is an **allowlist, not a floor**: The last two belong to the materialization layer and have no caller yet; they live here with the first so the allowlist is one section of one file rather than three sites to audit. +### The integrity-failure log record + +The signal above is product telemetry; the **log record** beside it is the customer-owned +detection path, and the more load-bearing of the two. It is the only integrity surface that +works when telemetry is off, and the only one that exists at all in an instance with no +telemetry destination, so it is a documented contract in the README rather than a debugging +aid. `record_integrity_failure` writes both, and is the only place either is constructed. + +One ERROR record per withheld skill, message text = `INTEGRITY_FAILURE_EVENT` + a space + +`json.dumps(record, sort_keys=True, separators=(",", ":"))`, plus the same mapping under +`extra={"ld_skills": record}`. Fields: `event`, `action` (always `withheld`), `skill_key`, +`version?`, `expected_hash?`, `observed_hash?`, `reason_code`, `reason`, `language`. + +Each of those choices is load-bearing; do not undo one as a simplification. + +- **The event name is in the message text**, not only in `extra`. Severity cannot + discriminate — `resolve_from_store` and `list_raw_objects` in the same module also log + ERROR for a raising store — and the stdlib's default formatter drops `extra` entirely, so + an `extra`-only record is invisible under a plain `logging.basicConfig()`. +- **`ld.skills.integrity_failure` is documented for customers to match on**, which makes it + a compatibility surface. It must never be renamed. +- **`sort_keys=True` is not cosmetic.** The other language implementations build the object + in alphabetical key order, so sorting makes the serialized line byte-identical across + SDKs for the same input, modulo `language`. +- **Optional fields are omitted, never nulled**, so a SIEM field-existence check means + something. +- **The record spreads the signal's properties** rather than rebuilding them, so the two + cannot drift on the fields they share — in particular on which are redacted. Anything + added later that comes off the wire needs the same shape-check-then-redact treatment. +- **`reason_code` is in the record only.** The signal's property set is the allowlist above + and does not grow; the local record is where the detection vocabulary lives. + +`reason_code` is a **closed vocabulary of exactly eight tokens** — `IntegrityReasonCode`, a +`Literal`, so a typo at a call site is a type error — one per `record_integrity_failure` +call site, and the same eight in every language implementation: + +| `reason_code` | Call site | +|---|---| +| `not_an_object` | `verify_raw_skill` — raw object is not a dict | +| `invalid_key` | `verify_raw_skill` — fails `is_valid_skill_key` | +| `invalid_version` | `verify_raw_skill` — fails `is_valid_skill_version` | +| `missing_content` | `verify_raw_skill` — `content` absent or not a string | +| `missing_content_hash` | `verify_raw_skill` — `contentHash` absent or not a string | +| `not_utf8` | `verified_bytes` — `UnicodeEncodeError` on encode (wire-`str` path only; a `Skill` already holds bytes) | +| `over_size_cap` | `verified_bytes` — over `MAX_SKILL_CONTENT_BYTES` | +| `hash_mismatch` | `verified_bytes` — observed sha256 != `contentHash` | + +Adding a ninth failure mode means widening `IntegrityReasonCode`, adding a case to +`REASON_CODE_CASES` in `test_skills.py` (whose exhaustiveness assertion fails otherwise), +documenting it in the README table, **and** doing the same in the other language SDKs. A +token added on one side only is a drift bug: a customer's detection rule stops matching +where they cannot see it. + `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and **deliberately excluded from SDK emission** — both are observable server-side. Do not add them. The skill body never appears in a signal, a log line, or @@ -431,4 +484,5 @@ on their side of the boundary. - `Skill.content` is opaque `bytes`. Do not add anything that parses or interprets it — no YAML library in this package's dependencies at any tier, and no accessor that decodes content. - Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. - Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. +- Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. - Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index c478be93..1e517e4e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -38,10 +38,11 @@ from __future__ import annotations import hashlib +import json import logging import re from dataclasses import dataclass -from typing import Any, Protocol +from typing import Any, Literal, Protocol, get_args from .types import Skill, SkillReference from .types_validation import is_valid_skill_key, is_valid_skill_version @@ -91,6 +92,46 @@ _SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" _SIGNAL_REVOKED = "AgentControl Skill Revoked Received" +INTEGRITY_FAILURE_EVENT = "ld.skills.integrity_failure" +""" +Stable event identity for the local integrity-failure log record. + +A **compatibility surface**, not an implementation detail: the README documents +it as the string a customer's SIEM matches on, so it must never be renamed. + +It appears verbatim **in the message text**, not only in ``extra``. Severity +alone cannot discriminate — ``list_raw_objects`` and ``resolve_from_store`` in +this module also log ERROR when a store raises — and the stdlib's default +formatter drops ``extra`` entirely, so under a plain ``logging.basicConfig()`` +an ``extra``-only record would be invisible. +""" + +_ACTION_WITHHELD = "withheld" +"""The only action an integrity failure results in: content is never returned.""" + +IntegrityReasonCode = Literal[ + "not_an_object", + "invalid_key", + "invalid_version", + "missing_content", + "missing_content_hash", + "not_utf8", + "over_size_cap", + "hash_mismatch", +] +""" +The closed ``reason_code`` vocabulary — one token per ``record_integrity_failure`` +call site, and the same eight tokens in every language implementation of this +feature, so a detection rule written against one SDK reads the others. + +A ``Literal`` rather than a bare ``str`` so a typo at a call site is a type +error, and so widening the vocabulary is a deliberate edit here rather than a +new string invented at the site that needed it. +""" + +INTEGRITY_REASON_CODES: frozenset[str] = frozenset(get_args(IntegrityReasonCode)) +"""``IntegrityReasonCode`` as a runtime set, derived rather than restated.""" + NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' @@ -234,17 +275,39 @@ def record_integrity_failure( skill_key: str, reason: str, *, + reason_code: IntegrityReasonCode, version: Any = None, expected_hash: Any = None, observed_hash: str | None = None, ) -> None: """ - Records an integrity failure. Carries hashes and byte counts only — the skill - body never appears in a signal, a log line, or an error message. + Records an integrity failure on both surfaces: one local log record, one + product signal. + + Carries hashes and byte counts only — the skill body never appears in a + signal, a log line, or an error message. + + The two surfaces are deliberately different sizes, and the log record is the + more important of the two. The signal is product telemetry: opt-out + respecting, no-op by default, and its property set is a documented allowlist + (``agents.md``) that does not grow. The **log record** is the customer-owned + detection path — the only one that works when telemetry is off, and the only + one that exists at all in an instance with no telemetry destination — so it + is designed to be ingested and alerted on, and additionally carries the + stable event name, the action taken, the human-readable reason, and the + machine-parseable ``reason_code``. + + Emitted twice over, because neither form alone is sufficient: the message + text carries ``INTEGRITY_FAILURE_EVENT`` followed by compact JSON, so the + record survives ``logging.basicConfig()`` and is greppable and ``jq``-able + under any handler configuration; ``extra["ld_skills"]`` carries the same + mapping unflattened for a structured handler that would rather not reparse. """ # Both of these come off the wire, so neither may be echoed verbatim: a store # that set contentHash (or key) to the skill body would otherwise publish the - # body itself. Shape-check, then redact. + # body itself. Shape-check, then redact. Every field the log record adds on + # top is either a literal or SDK-authored, so the record introduces no new + # untrusted value — anything added later needs this same treatment. safe_key = skill_key if is_valid_skill_key(skill_key) else "" properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} if is_valid_skill_version(version): @@ -258,7 +321,28 @@ def record_integrity_failure( if observed_hash is not None: properties["observed_hash"] = observed_hash - logger.error("Skill '%s' failed integrity verification: %s", safe_key, reason) + # Spread the signal's properties rather than rebuilding them, so the record + # cannot drift from the signal on the fields they share — in particular on + # which of them are redacted and which are omitted. Absent optional fields + # stay absent; the record never carries a null. + record: dict[str, Any] = { + "event": INTEGRITY_FAILURE_EVENT, + "action": _ACTION_WITHHELD, + "reason_code": reason_code, + "reason": reason, + **properties, + } + # ``sort_keys`` is load-bearing rather than cosmetic: the other language + # implementations build this object in alphabetical key order, so sorting + # here makes the serialized line byte-identical across SDKs for the same + # input, modulo ``language``. Do not drop it, and do not reorder the keys + # above expecting the output to follow. + logger.error( + "%s %s", + INTEGRITY_FAILURE_EVENT, + json.dumps(record, sort_keys=True, separators=(",", ":")), + extra={"ld_skills": record}, + ) emit(_SIGNAL_INTEGRITY_FAILURE, properties) @@ -375,7 +459,11 @@ def verified_bytes( # satisfy the hash comparison. reason = "content is not encodable as UTF-8" record_integrity_failure( - key, reason, version=version, expected_hash=expected_hash + key, + reason, + reason_code="not_utf8", + version=version, + expected_hash=expected_hash, ) return VerificationFailure(reason) @@ -385,7 +473,11 @@ def verified_bytes( f"{MAX_SKILL_CONTENT_BYTES} byte cap" ) record_integrity_failure( - key, reason, version=version, expected_hash=expected_hash + key, + reason, + reason_code="over_size_cap", + version=version, + expected_hash=expected_hash, ) return VerificationFailure(reason) @@ -396,6 +488,7 @@ def verified_bytes( record_integrity_failure( key, "content hash mismatch", + reason_code="hash_mismatch", version=version, expected_hash=expected_hash, observed_hash=observed_hash, @@ -414,7 +507,11 @@ def verify_raw_skill(raw: Any) -> Skill | None: user code. """ if not isinstance(raw, dict): - record_integrity_failure("", "raw skill object is not an object") + record_integrity_failure( + "", + "raw skill object is not an object", + reason_code="not_an_object", + ) return None key = raw.get("key") @@ -422,25 +519,34 @@ def verify_raw_skill(raw: Any) -> Skill | None: record_integrity_failure( key if isinstance(key, str) else "", "key is not a valid skill key", + reason_code="invalid_key", ) return None version = raw.get("version") if not is_valid_skill_version(version): - record_integrity_failure(key, "version is not an integer >= 1") + record_integrity_failure( + key, "version is not an integer >= 1", reason_code="invalid_version" + ) return None content = raw.get("content") if not isinstance(content, str): record_integrity_failure( - key, "content is missing or not a string", version=version + key, + "content is missing or not a string", + reason_code="missing_content", + version=version, ) return None expected_hash = raw.get("contentHash") if not isinstance(expected_hash, str): record_integrity_failure( - key, "contentHash is missing or not a string", version=version + key, + "contentHash is missing or not a string", + reason_code="missing_content_hash", + version=version, ) return None diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index e99a8957..203eeb4f 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1006,6 +1006,275 @@ async def test_store_error_does_not_leak_content( assert await all_skills() == [] +# --------------------------------------------------------------------------- +# Gap 1 — the local integrity-failure log record +# --------------------------------------------------------------------------- + +INTEGRITY_EVENT = "ld.skills.integrity_failure" +"""The stable event name, spelled out rather than imported. + +The name is a compatibility surface documented for customers to match on in a +SIEM, so the test has to fail when it is renamed. Importing the constant would +rename in lockstep and assert nothing. +""" + +LOGGED_BODY = "UNIQUE-SECRET-BODY-THAT-MUST-NOT-BE-LOGGED" + + +def _raw_object(**overrides: Any) -> dict[str, Any]: + """A wire-shaped raw object with a correct ``contentHash``. + + The ``make_raw_skill`` fixture is the same factory, but a module-level + ``parametrize`` table cannot reach a fixture. + """ + raw: dict[str, Any] = { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + raw.update(overrides) + return raw + + +def _raw_without(field: str) -> dict[str, Any]: + raw = _raw_object() + del raw[field] + return raw + + +_OVERSIZE = "x" * (64 * 1024 + 1) + +REASON_CODE_CASES = [ + # Not a dict at all. Reachable through ``all_skills`` and not through + # ``get_skill``, which rejects a non-dict answer before verification. + pytest.param("not-an-object-at-all", "not_an_object", id="not_an_object"), + pytest.param(_raw_object(key="Evil/../x"), "invalid_key", id="invalid_key"), + pytest.param(_raw_object(version=0), "invalid_version", id="invalid_version"), + pytest.param(_raw_without("content"), "missing_content", id="missing_content"), + pytest.param( + _raw_without("contentHash"), "missing_content_hash", id="missing_content_hash" + ), + # A lone surrogate has no UTF-8 encoding. Only reachable on the wire-``str`` + # path: a ``Skill`` already holds bytes and skips the encode. + pytest.param( + _raw_object(content=json.loads(r'"hi \ud800 there"')), "not_utf8", id="not_utf8" + ), + # Correct hash for the oversize body, so the cap is what withheld it. + pytest.param( + _raw_object(content=_OVERSIZE, contentHash=_hash(_OVERSIZE)), + "over_size_cap", + id="over_size_cap", + ), + pytest.param( + _raw_object(contentHash="0" * 64), "hash_mismatch", id="hash_mismatch" + ), +] +"""One case per ``reason_code`` token, driven end to end through ``all_skills``. + +``all_skills`` rather than ``get_skill`` for every case so the table is uniform: +it verifies every object the store holds, including the ones too malformed to +carry a usable key, which is the only accessor path a non-dict reaches. +""" + + +def _integrity_records(caplog: pytest.LogCaptureFixture) -> list[dict[str, Any]]: + """Every integrity-failure record in *caplog*, parsed out of the message text. + + Read off the message rather than off ``record.ld_skills`` deliberately: the + message is what a customer sees under a plain ``logging.basicConfig()``, and + it is the surface the documented contract is about. The structured mirror is + asserted separately, against this. + """ + parsed: list[dict[str, Any]] = [] + for entry in caplog.records: + message = entry.getMessage() + if not message.startswith(f"{INTEGRITY_EVENT} "): + continue + parsed.append(json.loads(message[len(INTEGRITY_EVENT) + 1 :])) + return parsed + + +class TestIntegrityFailureLogRecord: + """ + The local log record is a documented detection surface, not a debugging aid. + + It is the only integrity signal that survives telemetry being switched off, + and the only one that exists at all in an instance with no telemetry + destination, so its shape is a contract: a stable event name in the message + text, a closed ``reason_code`` vocabulary, and no field a hostile store can + dictate. + """ + + @pytest.fixture(autouse=True) + def _capture_errors(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level("ERROR", logger="launchdarkly_ai_server.skills_core") + + async def _withhold(self, objects: dict[str, Any]) -> None: + skills_module._set_store(InMemorySkillStore(objects)) + assert await all_skills() == [] + + @pytest.mark.parametrize(("raw", "expected_code"), REASON_CODE_CASES) + async def test_one_record_per_reason_code( + self, caplog: pytest.LogCaptureFixture, raw: Any, expected_code: str + ) -> None: + await self._withhold({"a": raw}) + + records = _integrity_records(caplog) + assert len(records) == 1 + record = records[0] + assert record["reason_code"] == expected_code + assert record["event"] == INTEGRITY_EVENT + assert record["action"] == "withheld" + assert record["language"] == "python" + assert record["reason"] # the human-readable half, carrying byte counts + # Absent optional fields are omitted, never nulled: a SIEM field + # existence check has to mean something. + assert None not in record.values() + # The body never reaches a log line. Swept over every failure mode here; + # the two cases below are the ones that make the rule observable, since + # a well-formed key and digest never enter a redaction branch. + assert "Do the thing." not in json.dumps(record) + + def test_the_case_table_exhausts_the_vocabulary(self) -> None: + """The vocabulary is closed, and every token in it is reachable. + + Both directions matter. A ninth token added to the source without a call + site fails here, and so does a ninth call site that invented a token the + table does not cover — which is what keeps the Python and TypeScript + vocabularies from drifting apart one edit at a time. + """ + from launchdarkly_ai_server import skills_core + + covered = {case.values[1] for case in REASON_CODE_CASES} + assert covered == skills_core.INTEGRITY_REASON_CODES + assert len(covered) == 8 + + async def test_the_event_name_is_in_the_message_text( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Severity alone cannot discriminate, so the name has to be in the line. + + ``resolve_from_store`` and ``list_raw_objects`` also log ERROR from this + logger when a store raises, and the stdlib's default formatter drops + ``extra`` entirely — an ``extra``-only record would be invisible to a + customer running ``logging.basicConfig()``. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + assert len(errors) == 1 + assert errors[0].getMessage().startswith(f"{INTEGRITY_EVENT} ") + + async def test_structured_handlers_get_the_same_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """``extra`` carries the record unflattened, and says the same thing.""" + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + # extra lands in the record's __dict__, which is where a + # structured handler reads it from. + assert errors[0].__dict__["ld_skills"] == _integrity_records(caplog)[0] + + async def test_redaction_survives_into_the_record( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Both wire-sourced fields are still redacted in the JSON payload. + + ``key`` and ``contentHash`` are attacker-controlled, so a store can put + the skill body in either. The record must not have reopened a leak the + signal already closed — same treatment, same placeholders. + """ + await self._withhold( + { + "a": { + "key": f"{LOGGED_BODY}/../x", + "version": 1, + "content": LOGGED_BODY, + "contentHash": LOGGED_BODY, + } + } + ) + + record = _integrity_records(caplog)[0] + assert record["skill_key"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_a_non_sha256_expected_hash_is_redacted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The key is valid here, so ``expected_hash`` is the field under test.""" + await self._withhold({"a": _raw_object(contentHash=LOGGED_BODY)}) + + record = _integrity_records(caplog)[0] + assert record["expected_hash"] == "" + assert LOGGED_BODY not in json.dumps(record) + + async def test_observed_hash_is_absent_before_hashing( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Nothing was hashed, so there is no observed value to report.""" + await self._withhold({"a": _raw_without("contentHash")}) + + assert "observed_hash" not in _integrity_records(caplog)[0] + + async def test_observed_hash_is_present_on_a_mismatch( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """The one case that is a possible active-tampering signal. + + Positive control for the test above: an implementation that never + populated ``observed_hash`` would satisfy it vacuously. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + record = _integrity_records(caplog)[0] + assert record["observed_hash"] == _hash(SKILL_BODY) + assert record["expected_hash"] == "0" * 64 + + async def test_the_serialized_payload_is_compact_and_key_sorted( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Sorted keys are what make the line byte-identical across SDKs. + + The other language implementations build this object in alphabetical + order, so a Python line following insertion order would differ byte for + byte on identical input. Compact separators are asserted alongside + because the two together are the serialization contract. + """ + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + errors = [r for r in caplog.records if r.levelname == "ERROR"] + payload = errors[0].getMessage()[len(INTEGRITY_EVENT) + 1 :] + # json.loads preserves the document's order, so this is what was written. + keys = list(json.loads(payload)) + assert keys == sorted(keys) + assert ", " not in payload and ": " not in payload + + async def test_reason_code_stays_out_of_the_telemetry_signal( + self, recording_emitter: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The signal's property set is an allowlist and does not grow. + + The record is the customer-owned detection path and carries the new + vocabulary; the LD-side counter is product telemetry and was left + exactly as designed. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._withhold({"a": _raw_object(contentHash="0" * 64)}) + + props = recording_emitter.signals(INTEGRITY_SIGNAL)[0] + assert set(props) == { + "skill_key", + "version", + "expected_hash", + "observed_hash", + "language", + } + assert _integrity_records(caplog)[0]["reason_code"] == "hash_mismatch" + + class TestTelemetrySeam: """Internal emitter seam, no client.track, no context.""" From 8ad49f21b614022306436687d3ca65bdac0efa80 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:06:16 -0400 Subject: [PATCH 04/43] feat(client): descriptor-pinned filesystem primitives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third of five slices. Adds `safe_fs.py`, the "write a file under a directory something else may be racing you for" problem solved once. Nothing here knows what a skill is; the materialization layer is its only caller, and it lands next. A path check is only as good as the last path resolution after it. Every `lstat` and containment check validates an inode, but a following `os.replace(tmp, dir / name)` re-resolves `dir` from its name — so anything holding write permission there can move the validated directory aside, leave a symlink in its place, and redirect the write or the unlink somewhere else. Narrowing that window is not a fix; the race is winnable at any width. So the checks hand off to a descriptor and nothing re-resolves a path afterwards. - `open_directory_nofollow` opens with `O_RDONLY | O_DIRECTORY | O_NOFOLLOW` and confirms `S_ISDIR` on the `fstat`, since not every platform defines `O_DIRECTORY`. `open_or_create_directory` adds `os.mkdir` plus an `lstat` on the `FileExistsError` path, because `Path.mkdir(exist_ok=True)` accepts a symlink-to-directory as "already there" and would reopen the hole the caller's check just closed. `pinned_directory` holds either for a block, so a caller states the platform split once and cannot forget the close. - `atomic_write` creates its temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` at that descriptor, `fchmod`s the descriptor rather than a path, writes, fsyncs, renames, and fsyncs the directory so the rename survives a crash. Mode is set explicitly at 0644, never inherited from the umask and never executable. `os.replace` is the single rename call site and `os.rename` must not be substituted for it. - `unlink_file` probes and unlinks descriptor-relative. `unlink` never follows a trailing symlink but it does resolve the directory above it, so the same swap turns a removal into a delete of an arbitrary file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away — the state on disk is not what the caller believes, and that is the caller's to report. `SUPPORTS_DIR_FD` gates all of it, and the probe deliberately names `os.rename`/`os.stat` rather than the `os.replace`/`os.lstat` this module calls: `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` under `rename` only and `fstatat` under `stat` only. Probing the names actually called reports "unsupported" on every POSIX platform and silently turns the defense off. Where the family is absent, every operation falls back to the identical full-path sequence. The tests here exercise the module directly, on its own terms. The TOCTOU races these primitives exist to close are proved through the materialization layer, which is what holds a descriptor across a sequence of operations. Testing: `uv run pytest` → 1260 passed, 11 skipped. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/agents.md | 47 +++ .../src/launchdarkly_ai_server/safe_fs.py | 311 ++++++++++++++++++ packages/client/tests/test_safe_fs.py | 246 ++++++++++++++ 3 files changed, 604 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/safe_fs.py create mode 100644 packages/client/tests/test_safe_fs.py diff --git a/packages/client/agents.md b/packages/client/agents.md index b5733567..152f5771 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -334,6 +335,52 @@ The injection path is deliberately narrower than the state's location: `skills.p (`skills._set_store(store)` is the same setter `init_client` uses), and neither should reach into `skills_core` directly. +### Descriptor-pinned filesystem access + +A path check is only as good as the last path resolution after it. Every `lstat`, `realpath` +and containment check validates an *inode*, but a following +`os.replace(tmp, root / key / "SKILL.md")` re-resolves `/` from its *name* — so +anything holding write permission on a managed directory can move the validated directory +aside, leave a symlink in its place, and redirect the write (or an unlink) somewhere else. +Narrowing that window is not a fix; the race is winnable at any width. + +So the checks hand off to a descriptor and nothing re-resolves a path afterwards. The +primitives live in `safe_fs.py`, which knows nothing about skills: + +- `open_directory_nofollow` opens the directory with `O_RDONLY | O_DIRECTORY | O_NOFOLLOW` + and confirms `S_ISDIR` on the `fstat` (the explicit check is what covers platforms with no + `O_DIRECTORY`). `open_or_create_directory` wraps it with `os.mkdir` plus an `lstat` on the + `FileExistsError` path — `Path.mkdir(exist_ok=True)` accepts a symlink-to-directory as + "already there", which would reopen the hole the caller's check just closed. + `pinned_directory` holds either for the duration of a block, so a caller states the + platform split once as `if dir_fd is not None` and cannot forget the `os.close`. +- `atomic_write` creates the temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` **at** that + descriptor (`_mkstemp_at`, since `tempfile` has no `dir_fd` form), `fchmod`s the + descriptor rather than `chmod`ing a path, writes, fsyncs, and renames with + `os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)`, then fsyncs the directory so the + rename survives a crash. `atomic_write_in` is the same against a directory the caller does + not already hold open. `os.replace` is the single rename call site, reached by attribute + lookup so tests can intercept it, and `os.rename` must not be substituted for it — it is + also the only one with defined overwrite semantics on Windows. +- `unlink_file` probes and unlinks descriptor-relative too. `unlink` never follows a + *trailing* symlink, but it does resolve the directory above it, so the same swap turns a + removal into a delete of an attacker-chosen file. A symlink found where this SDK expects + its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is + not what the caller believes, and that is the caller's to report. + +`safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. +`os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` +under `os.rename` only and `fstatat` under `os.stat` only — even though `os.replace` is the +same `renameat`-backed function and `os.lstat` is `fstatat` with `AT_SYMLINK_NOFOLLOW`. +Probing the names this module actually calls reports "unsupported" on every POSIX platform +and silently turns the defense off, so the probe names the advertised twins +(`{os.rename, os.open, os.unlink, os.stat}`) and a caller's symlink check is spelled +`os.stat(..., follow_symlinks=False)` rather than `os.lstat`. Where the family is absent +(Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of +attempting the descriptor open — `os.open` cannot open a directory there — and every caller +falls back to the identical full-path sequence, the per-component `lstat` floor. The +residual window on those platforms is documented rather than closed. + --- ## OTel Setup diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py new file mode 100644 index 00000000..3605399e --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -0,0 +1,311 @@ +""" +Descriptor-pinned filesystem primitives. + +Split out because none of this knows what a skill is: it is the "write a file +under a directory an attacker may be racing you for" problem, solved once. +``skills_fs.py`` is the only caller today. + +The whole point is that a path check is only as good as the last path +resolution after it. Every operation here therefore runs relative to a +descriptor pinned to a directory the caller has already validated, rather than +re-resolving a name — which is what closes the swap window rather than merely +narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the +identical sequence runs against full paths, the per-component ``lstat`` floor. +""" + +from __future__ import annotations + +import errno +import os +import secrets +import stat +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +_FILE_MODE = 0o644 +"""Mode set explicitly on every written file — never inherited from the umask, +and never executable.""" + +SUPPORTS_DIR_FD = os.supports_dir_fd.issuperset( + # renameat, openat, unlinkat, fstatat — the four this module needs. + {os.rename, os.open, os.unlink, os.stat} +) +""" +Whether the ``*at()`` syscall family is available, so every operation under the +managed root can be performed relative to a descriptor pinned to a directory +this module has already verified rather than re-resolved from its path. + +That is what closes the swap window rather than merely narrowing it: +a descriptor refers to the inode that was checked, so replacing ``/`` +with a symlink after the check cannot redirect a write or an unlink out of the +root. POSIX has these calls; Windows does not, and there the per-component +``lstat`` floor the spec permits applies instead. + +The probe deliberately names ``os.rename`` and ``os.stat`` rather than the +``os.replace`` and ``os.lstat`` this module actually calls. ``os.supports_dir_fd`` +is populated per underlying syscall, and CPython registers ``renameat`` under +``rename`` only and ``fstatat`` under ``stat`` only — even though ``os.replace`` +is the same ``renameat``-backed function and ``os.lstat`` is ``fstatat`` with +``AT_SYMLINK_NOFOLLOW``, and both accept the descriptor keywords wherever their +advertised twin does (verified on CPython 3.12 and 3.13, macOS). Probing the +names this module calls would report "unsupported" on every POSIX platform and +silently disable the defense. +""" + + +def open_directory_nofollow(directory: Path) -> int | None: + """ + Opens *directory* without following a final symlink, and pins it. + + Everything the caller does afterwards goes through the returned descriptor + instead of the path, which is what turns the "narrow window" into no + window at all: the descriptor names the inode that was checked, so swapping + the path for a symlink between the check and the write cannot redirect the + write out of the managed root. + + On a platform without the ``*at()`` family (Windows) this returns ``None`` + after verifying via ``lstat`` that the path is a real, non-symlink + directory — the per-component floor. It must not attempt the descriptor + open there: ``os.open`` goes through the CRT on Windows, which cannot open + a directory at all, so the descriptor path would fail every operation + rather than fall back. + + Raises ``ValueError`` when the path will not open (or inspect) as a real + directory — the caller reports that as a refusal rather than letting it + escape. + """ + if not SUPPORTS_DIR_FD: + try: + mode = os.lstat(directory).st_mode + except OSError as exc: + raise ValueError(f"the directory could not be inspected: {exc}") from exc + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") + if not stat.S_ISDIR(mode): + raise ValueError("the path is not a directory") + return None + + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(directory, flags) + except OSError as exc: + raise ValueError( + f"the directory could not be opened without following links: {exc}" + ) from exc + try: + # O_DIRECTORY already guarantees this wherever the platform defines it; + # the explicit check is what covers the platforms that do not. + if not stat.S_ISDIR(os.fstat(fd).st_mode): + raise ValueError("the path is not a directory") + except BaseException: + os.close(fd) + raise + return fd + + +def open_or_create_directory(directory: Path) -> int | None: + """ + Creates *directory* if absent and returns a descriptor pinned to it. + + ``Path.mkdir(exist_ok=True)`` treats an existing symlink-to-directory as + "already there", which would re-open the very hole the caller's ``lstat`` + check just closed. ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` + path does not: a link reports as a link, and is refused. + """ + try: + os.mkdir(directory, 0o755) + except FileExistsError: + mode = os.lstat(directory).st_mode + if stat.S_ISLNK(mode): + raise ValueError("the directory is a symlink") from None + if not stat.S_ISDIR(mode): + raise ValueError("the path is not a directory") from None + return open_directory_nofollow(directory) + + +@contextmanager +def pinned_directory(directory: Path, *, create: bool = False) -> Iterator[int | None]: + """ + Holds *directory* pinned for the duration of the block, then releases it. + + Yields what the two openers above return — a descriptor, or ``None`` on the + ``lstat`` floor — so the caller states the platform split once, as + ``if dir_fd is not None``, and cannot forget the ``os.close``. Raises + ``ValueError`` for a directory that will not pin, exactly as they do. + """ + dir_fd = ( + open_or_create_directory(directory) + if create + else open_directory_nofollow(directory) + ) + try: + yield dir_fd + finally: + if dir_fd is not None: + os.close(dir_fd) + + +class SymlinkRefused(OSError): + """ + Raised instead of removing a symlink found where a real file was expected. + + An ``OSError`` subclass so a caller that only cares that the removal failed + keeps its single ``except``; a distinct type so one that must report *this* + refusal specifically does not have to match on a message. + """ + + +def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: + """ + Removes ``/``, refusing to follow a symlink at *name*. + + The mirror of ``atomic_write``, and descriptor-relative for the same reason: + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so a ```` swapped for a symlink after the + caller's checks would otherwise turn this into a delete of an + attacker-chosen file. Given a *dir_fd* the probe and the unlink both run + against it; without one the identical sequence runs against full paths. + + Raises ``SymlinkRefused`` when *name* is a symlink. Note that this refuses + rather than removes: ``unlink`` would happily delete the link itself, but a + link where this SDK expects its own file means the state on disk is not what + the manifest describes, and that is the caller's to report rather than to + tidy away. + """ + if dir_fd is None: + # No ``*at()`` family: the trailing-symlink check and the unlink are both + # path-based, the per-component floor. + target = directory / name + if target.is_symlink(): + raise SymlinkRefused(f"{name} is a symlink") + target.unlink() + return + + # os.stat(follow_symlinks=False), not os.lstat: identical result, and it is + # the spelling os.supports_dir_fd actually advertises. + probe = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) + if stat.S_ISLNK(probe.st_mode): + raise SymlinkRefused(f"{name} is a symlink") + os.unlink(name, dir_fd=dir_fd) + + +def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: + """ + ``tempfile.mkstemp`` for a directory descriptor. + + ``tempfile`` has no ``dir_fd`` form, so this reproduces the part that + matters: ``O_CREAT | O_EXCL`` against an unpredictable name, retried on + collision, so an existing temp path is never reused and a planted one is + never written through. + """ + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + for _ in range(tempfile.TMP_MAX): + name = f"{prefix}{secrets.token_hex(8)}.tmp" + try: + return os.open(name, flags, 0o600, dir_fd=dir_fd), name + except FileExistsError: + continue + raise OSError(errno.EEXIST, "no usable temporary file name was found") + + +def atomic_write( + directory: Path, name: str, data: bytes, *, dir_fd: int | None = None +) -> None: + """ + Writes *data* to ``/`` so no partial file is ever + observable. + + The temp file is created exclusively in the target's *own* directory — one + anywhere else would make the rename cross-device, and therefore not atomic — + written, fsynced, renamed over the target, and the directory fsynced so the + rename itself survives a crash. Mode is set explicitly rather than left to + the process umask, and the execute bit is never set. + + Given a *dir_fd* on a platform with the ``*at()`` family, every one of those + steps runs relative to that descriptor and both names are bare filenames. + Without one (Windows) the identical sequence runs against full paths, which + is the per-component ``lstat`` floor. + + ``os.replace`` is the one and only rename call site, reached by attribute + lookup on the ``os`` module so tests can intercept it; ``os.rename`` must + not be substituted for it (it is also the only one with defined overwrite + semantics on Windows). + """ + at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None + prefix = f".{name}." + target: str | Path + + if at_fd is not None: + fd, temp = _mkstemp_at(at_fd, prefix) + target = name + else: + # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never + # reused. + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=".tmp") + target = directory / name + + try: + try: + # fchmod, not chmod: operating on the descriptor cannot be redirected + # by anything that swaps the temp path underneath us, and it makes the + # mode independent of the process umask (both creation paths open 0600). + os.fchmod(fd, _FILE_MODE) + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + os.fsync(fd) + finally: + os.close(fd) + if at_fd is not None: + os.replace(temp, target, src_dir_fd=at_fd, dst_dir_fd=at_fd) + else: + os.replace(temp, target) + except BaseException: + try: + if at_fd is not None: + os.unlink(temp, dir_fd=at_fd) + else: + os.unlink(temp) + except OSError: + pass + raise + + if at_fd is not None: + _fsync_directory_fd(at_fd) + else: + _fsync_directory(directory) + + +def atomic_write_in(directory: Path, name: str, data: bytes) -> None: + """ + ``atomic_write`` against a directory this module does not already hold open. + + Used for the skills manifest, whose directory is the managed root. The + descriptor is taken with ``O_NOFOLLOW``, so a root swapped for a symlink after + ``_resolve_root`` validated it fails the write instead of redirecting it — + the caller turns that into a run-level ``error`` action. + """ + with pinned_directory(directory) as dir_fd: + atomic_write(directory, name, data, dir_fd=dir_fd) + + +def _fsync_directory_fd(fd: int) -> None: + """Best effort — not every platform allows fsync on a directory descriptor.""" + try: + os.fsync(fd) + except OSError: + pass + + +def _fsync_directory(directory: Path) -> None: + """Best effort — not every platform lets a directory be opened for fsync.""" + try: + fd = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + _fsync_directory_fd(fd) + finally: + os.close(fd) diff --git a/packages/client/tests/test_safe_fs.py b/packages/client/tests/test_safe_fs.py new file mode 100644 index 00000000..b148ff9b --- /dev/null +++ b/packages/client/tests/test_safe_fs.py @@ -0,0 +1,246 @@ +""" +Tests for the descriptor-pinned filesystem primitives. + +These exercise ``safe_fs`` directly, on its own terms — the module knows nothing +about skills, and its guarantees are worth asserting without a caller in the way. +The TOCTOU races these primitives exist to close are proved through the +materialization layer, which is what actually holds a descriptor across a +sequence of operations. +""" + +from __future__ import annotations + +import os +import stat +from pathlib import Path + +import pytest + +import launchdarkly_ai_server.safe_fs as safe_fs_module +from launchdarkly_ai_server.safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + open_directory_nofollow, + open_or_create_directory, + pinned_directory, + unlink_file, +) + + +class TestOpenDirectory: + """Pinning a directory, and refusing anything that is not one.""" + + def test_opens_a_real_directory(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + assert stat.S_ISDIR(os.fstat(dir_fd).st_mode) + + def test_refuses_a_symlink_to_a_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_directory_nofollow(link) + + def test_refuses_a_regular_file(self, tmp_path: Path) -> None: + target = tmp_path / "file" + target.write_text("not a directory") + with pytest.raises(ValueError): + open_directory_nofollow(target) + + def test_refuses_an_absent_path(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + open_directory_nofollow(tmp_path / "nope") + + def test_create_makes_the_directory(self, tmp_path: Path) -> None: + target = tmp_path / "new" + fd = open_or_create_directory(target) + try: + assert target.is_dir() + finally: + if fd is not None: + os.close(fd) + + def test_create_refuses_an_existing_symlink(self, tmp_path: Path) -> None: + """``Path.mkdir(exist_ok=True)`` would accept this and reopen the hole. + + A symlink-to-directory already present reads as "already there" to + ``exist_ok``, so the caller's containment check would be bypassed by + something that was never checked. ``os.mkdir`` plus an ``lstat`` on the + ``FileExistsError`` path refuses it. + """ + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + open_or_create_directory(link) + + def test_pinned_directory_closes_the_descriptor(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + if dir_fd is None: + pytest.skip("no *at() family on this platform") + held = dir_fd + with pytest.raises(OSError): + os.fstat(held) + + +class TestAtomicWrite: + """Explicit mode, no observable partial file, and one rename call site.""" + + def test_writes_the_bytes(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"hello", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"hello" + + def test_mode_is_0644_and_never_executable(self, tmp_path: Path) -> None: + """Set explicitly on the descriptor, so the process umask cannot widen or + narrow it and the execute bit is never inherited.""" + previous = os.umask(0o077) + try: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + finally: + os.umask(previous) + mode = (tmp_path / "f.txt").stat().st_mode + assert stat.S_IMODE(mode) == 0o644 + assert not mode & stat.S_IXUSR + + def test_overwrites_an_existing_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_bytes(b"old") + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"new", dir_fd=dir_fd) + assert (tmp_path / "f.txt").read_bytes() == b"new" + + def test_leaves_no_temp_file_behind(self, tmp_path: Path) -> None: + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_a_failed_rename_removes_the_temp_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A crash between write and rename must not leave a partial file, and + must not leave the temp file either.""" + + def _boom(*args: object, **kwargs: object) -> None: + raise OSError("injected rename failure") + + monkeypatch.setattr(os, "replace", _boom) + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(OSError, match="injected"): + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert list(tmp_path.iterdir()) == [] + + def test_rename_goes_through_os_replace( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``os.replace`` is the single rename call site. + + ``os.rename`` must not be substituted for it: it is the only one with + defined overwrite semantics on Windows, and it is the seam the + materialization tests intercept to prove atomicity. + """ + calls: list[object] = [] + real = os.replace + + def _spy(src: object, dst: object, **kwargs: object) -> None: + calls.append(dst) + real(src, dst, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(os, "replace", _spy) + with pinned_directory(tmp_path) as dir_fd: + atomic_write(tmp_path, "f.txt", b"x", dir_fd=dir_fd) + assert len(calls) == 1 + assert os.path.basename(str(calls[0])) == "f.txt" + + def test_write_without_a_descriptor_uses_the_path_fallback( + self, tmp_path: Path + ) -> None: + """The no-``*at()`` shape must produce an identical result. + + Windows takes this path for every write, so it is not a degenerate case — + the file, its mode, and the absence of a temp file all have to match. + """ + atomic_write(tmp_path, "f.txt", b"fallback", dir_fd=None) + assert (tmp_path / "f.txt").read_bytes() == b"fallback" + assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + + def test_write_in_pins_the_directory_itself(self, tmp_path: Path) -> None: + atomic_write_in(tmp_path, "f.txt", b"x") + assert (tmp_path / "f.txt").read_bytes() == b"x" + + def test_write_in_refuses_a_symlinked_directory(self, tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + with pytest.raises(ValueError): + atomic_write_in(link, "f.txt", b"x") + + +class TestUnlinkFile: + """Removing only a real file, and refusing a link found in its place.""" + + def test_removes_a_regular_file(self, tmp_path: Path) -> None: + (tmp_path / "f.txt").write_text("x") + with pinned_directory(tmp_path) as dir_fd: + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + assert not (tmp_path / "f.txt").exists() + + def test_refuses_a_symlink_rather_than_removing_it(self, tmp_path: Path) -> None: + """It refuses rather than tidies. + + ``unlink`` would happily delete the link itself, but a link where this SDK + expects its own file means the state on disk is not what the manifest + describes — the caller's to report, not this module's to clean up. + """ + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + link = tmp_path / "f.txt" + link.symlink_to(outside) + + with pinned_directory(tmp_path) as dir_fd: + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=dir_fd) + + assert link.is_symlink() + assert outside.read_text() == "do not touch" + + def test_refuses_a_symlink_on_the_path_fallback(self, tmp_path: Path) -> None: + outside = tmp_path / "outside.txt" + outside.write_text("do not touch") + (tmp_path / "f.txt").symlink_to(outside) + with pytest.raises(SymlinkRefused): + unlink_file(tmp_path, "f.txt", dir_fd=None) + assert outside.exists() + + def test_symlink_refused_is_an_oserror(self) -> None: + """A caller that only cares the removal failed keeps its single + ``except OSError``; one that must report this refusal specifically does + not have to match on a message.""" + assert issubclass(SymlinkRefused, OSError) + + +class TestDirFdProbe: + """The capability probe names the advertised twins, not the calls made.""" + + def test_probe_names_the_syscalls_python_advertises(self) -> None: + """``os.supports_dir_fd`` is populated per underlying syscall, and CPython + registers ``renameat`` under ``os.rename`` and ``fstatat`` under + ``os.stat``. Probing ``os.replace`` and ``os.lstat`` — the names this + module actually calls — reports "unsupported" on every POSIX platform and + would silently disable the defense. + """ + expected = os.supports_dir_fd.issuperset( + {os.rename, os.open, os.unlink, os.stat} + ) + assert safe_fs_module.SUPPORTS_DIR_FD is expected + + @pytest.mark.skipif(os.name == "nt", reason="POSIX advertises the *at() family") + def test_posix_has_the_family(self) -> None: + assert safe_fs_module.SUPPORTS_DIR_FD is True From f1aad0afb7629d231fcffde5159ada136f586032 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:12:17 -0400 Subject: [PATCH 05/43] =?UTF-8?q?test(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20filesystem=20abuse=20matrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth and last of five slices. The adversary. Every filesystem defense the previous two slices introduced now has a test that fails if the defense is removed, plus the materialization telemetry allowlist. - **Path traversal.** A key that escapes the root, a key that is the manifest filename, an over-long key, and a key that resolves outside after `realpath` — each refused before any filesystem call, and the resolved containment check asserted on inode identity rather than on path strings. - **Symlink attacks.** A symlinked skill directory, a symlinked target file, and the `/` directory swapped for a symlink at the exact instant of the rename and of the unlink — the narrowest version of the window the descriptor pinning exists to close, fired from the interception point rather than from implementation internals. - **The no-`*at()` shape.** The path fallback Windows takes for every operation, exercised with the capability probe forced off, so the platform that cannot pin a descriptor is not the untested one. The TOCTOU tests skip off that same flag deliberately: a probe that wrongly reported "unsupported" cannot also silently skip the tests that would have caught it. - **Non-regular files and clobber protection.** A fifo or a directory where `SKILL.md` belongs, and a file at a managed path with no matching manifest entry — reported and left alone, never overwritten and never removed. - **Corrupt manifests.** Unreadable, unparseable, not an object, malformed entries, and a `manifestVersion` this release cannot read: no overwrites, no prunes, an error action naming the manifest, and the manifest itself left as it was found. - **Atomicity.** A crash injected between the write and the rename leaves neither a partial file nor a temp file, and the one recorded rename is proved to have moved `SKILL.md` within the target's own directory — by descriptor identity where the platform has `renameat`, which also rules out the descriptor having been redirected between the check and the rename. - **Telemetry.** The three signal names are asserted as an allowlist rather than a floor: any other name reaching the emitter fails, the two deliberately excluded names are called out by name, no signal carries a filesystem path or the skill body, an emitter that raises never fails the reconcile, and `client.track()` is never reached. Two of these are worth naming, because the obvious test does not reach the guard. The unencodable-content cases pin `contentHash` to the sha256 of the bytes a non-strict encoder would have fabricated, since an arbitrary wrong hash is rejected by the mismatch check first and never exercises the encoder guard. The redaction cases smuggle the body through `contentHash` and through `key`, since a sweep using a well-formed digest under a valid key reaches neither replacement branch. Testing: `uv run pytest` → 1280 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/tests/test_skills_fs.py | 1071 ++++++++++++++++++++++- 1 file changed, 1064 insertions(+), 7 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..6502aa5b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1,13 +1,9 @@ """ -Tests for ``write_skills`` — filesystem materialization and manifest reconcile -semantics. +Tests for ``write_skills`` — filesystem materialization, manifest reconcile +semantics, and the full security abuse matrix. Every test writes only inside pytest's ``tmp_path``. No network, no real LaunchDarkly client, no real skill transport. - -The security abuse matrix — path traversal, symlink attacks, clobber -protection, corrupt manifests, atomicity under an injected crash, and the -materialization telemetry allowlist — is a separate module. """ from __future__ import annotations @@ -15,24 +11,195 @@ import hashlib import json import os +import stat from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import pytest +import launchdarkly_ai_server.safe_fs as safe_fs_module import launchdarkly_ai_server.skills as skills_module +import launchdarkly_ai_server.skills_fs as skills_fs_module from launchdarkly_ai_server import ( InMemorySkillStore, Skill, SkillReference, + get_skill, + init_client, write_skills, ) MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +MATERIALIZED_SIGNAL = "AgentControl Skill Materialized" +REVOKED_SIGNAL = "AgentControl Skill Revoked Received" INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" +# The three signal names are an allowlist, not a floor. +APPROVED_SIGNALS = frozenset({MATERIALIZED_SIGNAL, REVOKED_SIGNAL, INTEGRITY_SIGNAL}) + +# Considered and deliberately excluded from SDK emission — named explicitly +# so the regression is unmissable. +REMOVED_SIGNALS = frozenset( + { + "AgentControl Skill SDK Reference Returned", + "AgentControl Skill Content Retrieved", + } +) + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +_INJECTED = "simulated crash between write and rename" + + +def _dir_id(path: Path) -> tuple[int, int]: + """``(st_dev, st_ino)`` — a directory's identity, independent of its name.""" + info = os.stat(path) + return (info.st_dev, info.st_ino) + + +class _RenameCall(NamedTuple): + """One intercepted ``os.replace`` of a ``SKILL.md``. + + ``src``/``dst`` are exactly what the implementation passed. Where the rename + is ``dir_fd``-relative they are bare filenames and the location lives in the + descriptors, so ``*_dir_id`` carries each descriptor's ``(st_dev, st_ino)`` + resolved *at call time* — the implementation closes the descriptors as soon + as the write returns, so they cannot be resolved from the assertions. + """ + + src: str + dst: str + src_dir_fd: int | None + dst_dir_fd: int | None + src_dir_id: tuple[int, int] | None + dst_dir_id: tuple[int, int] | None + + +class _ReplaceSpy: + """Records — and optionally fails — every atomic rename of a ``SKILL.md``. + + Write/rename interception hook: the implementation performs + the final rename through a single ``os.replace`` call site, so patching the + attribute on the ``os`` module observes it. Destinations other than + ``SKILL.md`` (i.e. the manifest's own atomic write) pass straight through — + the filter holds for both call shapes, since the ``dir_fd``-relative form + passes ``"SKILL.md"`` itself as ``dst``. + + Used two ways: to prove an injected failure is what produced an ``error`` + action (atomicity), and to prove no write was *attempted* for a + rejected key — the OS would reject several hostile keys on its + own, so a failed write is not evidence of a defense. + """ + + def __init__(self, fail: bool = False) -> None: + self.calls: list[_RenameCall] = [] + self._fail = fail + self._real = os.replace + + def __call__(self, src: Any, dst: Any, **kwargs: Any) -> None: + if str(dst).endswith("SKILL.md"): + src_dir_fd = kwargs.get("src_dir_fd") + dst_dir_fd = kwargs.get("dst_dir_fd") + self.calls.append( + _RenameCall( + src=str(src), + dst=str(dst), + src_dir_fd=src_dir_fd, + dst_dir_fd=dst_dir_fd, + src_dir_id=None if src_dir_fd is None else _fd_id(src_dir_fd), + dst_dir_id=None if dst_dir_fd is None else _fd_id(dst_dir_fd), + ) + ) + if self._fail: + raise OSError(_INJECTED) + self._real(src, dst, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _ReplaceSpy: + # The attribute is set on the shared ``os`` module, so the + # single ``os.replace`` call site in safe_fs is intercepted wherever it is + # reached from. Named through the calling module rather than an arbitrary + # one so the hook documents which code it covers. + monkeypatch.setattr(safe_fs_module.os, "replace", self) + return self + + +def _fd_id(fd: int) -> tuple[int, int]: + info = os.fstat(fd) + return (info.st_dev, info.st_ino) + + +def _assert_atomic_rename_of(spy: _ReplaceSpy, skill_dir: Path) -> None: + """Assert the one recorded rename put ``SKILL.md`` into *skill_dir*. + + The temp file must be created in the target's own + directory, so the rename is atomic rather than cross-device. Two call + shapes prove it. Where the platform has ``renameat`` + the rename is ``dir_fd``-relative and the property is asserted by descriptor + identity — one descriptor for both sides, resolving to *skill_dir*'s inode — + which is stronger than comparing path strings, because it also rules out the + descriptor having been redirected between the check and the rename. On the + ``lstat`` floor (Windows) the names are full paths and share a parent. + """ + assert len(spy.calls) == 1 + call = spy.calls[0] + + if safe_fs_module.SUPPORTS_DIR_FD: + assert call.dst == "SKILL.md" + assert call.src != "SKILL.md" + assert call.src_dir_fd is not None + assert call.src_dir_fd == call.dst_dir_fd + assert call.dst_dir_id == _dir_id(skill_dir) + else: + assert Path(call.dst) == skill_dir / "SKILL.md" + assert Path(call.src).parent == skill_dir + assert Path(call.src).name != "SKILL.md" + + +class _SwapDirectoryDuring: + """Fires the directory-swap race at the exact instant of an operation. + + Renames ``/`` aside and leaves a symlink to *outside* in its + place, then lets the intercepted call proceed — the narrowest possible + version of the window an attacker with write access to the managed root + would otherwise have to hit by timing. Both hooks are the + interception points (``os.replace`` for the write, ``os.unlink`` for the + prune), so no implementation internals are touched. + """ + + def __init__(self, attribute: str, skill_dir: Path, outside: Path) -> None: + self.attribute = attribute + self.skill_dir = skill_dir + self.moved_to = skill_dir.parent / f"{skill_dir.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + named = args[0] if args else first + if str(named).endswith("SKILL.md") and not self.swapped: + os.rename(self.skill_dir, self.moved_to) + os.symlink(self.outside, self.skill_dir, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapDirectoryDuring: + # ``os.replace`` is called from safe_fs, ``os.unlink`` from skills_fs; both + # resolve to the same module object, so either name reaches both. + module = safe_fs_module if self.attribute == "replace" else skills_fs_module + monkeypatch.setattr(module.os, self.attribute, self) + return self + + +_needs_dir_fd = pytest.mark.skipif( + not safe_fs_module.SUPPORTS_DIR_FD, + reason="no *at() family on this platform; the per-component lstat floor applies", +) + @pytest.fixture def root(tmp_path: Path) -> Path: @@ -580,3 +747,893 @@ async def test_mismatch_does_not_disturb_existing_managed_file( await write_skills([bad], root) assert target.read_text(encoding="utf-8") == SKILL_BODY + + +class TestAtomicityAndPermissions: + """Atomic writes, no partial files, 0644.""" + + async def test_written_file_is_0644_and_not_executable(self, root: Path) -> None: + await write_skills([_skill("a")], root) + mode = stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) + assert mode == 0o644 + assert not mode & stat.S_IXUSR + assert not mode & stat.S_IXGRP + assert not mode & stat.S_IXOTH + + async def test_write_goes_through_a_single_atomic_rename( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the interception hook. + + Without this, the ``spy.calls == []`` assertions in the failure tests + below and in the traversal matrix could pass in a suite where the hook + is never reachable at all. + """ + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + # The temp file is created in the *same* directory + # as the target, so the rename is atomic rather than cross-device. + _assert_atomic_rename_of(spy, root / "a") + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_rename_failure_leaves_prior_content_intact( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "brand new content\n")], root) + + # The injected failure — not an unrelated rejection, and not an + # implementation that attempted nothing — is what produced the error. + _assert_atomic_rename_of(spy, target.parent) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert _INJECTED in (action.error or "") + + assert target.read_text(encoding="utf-8") == SKILL_BODY + # No temp artifact survives the failed run. + assert sorted(p.name for p in target.parent.iterdir()) == ["SKILL.md"] + + async def test_no_partial_file_at_target_after_failure( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy(fail=True).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + _assert_atomic_rename_of(spy, root / "a") + + assert report.ok is False + assert _INJECTED in (_actions_by_key(report)["a"].error or "") + assert not (root / "a" / "SKILL.md").exists() + # Neither a partial target nor a leaked temp file. + skill_dir = root / "a" + leftovers = ( + sorted(p.name for p in skill_dir.iterdir()) if skill_dir.exists() else [] + ) + assert leftovers == [] + + async def test_manifest_is_valid_json_after_a_run_with_errors( + self, root: Path + ) -> None: + report = await write_skills([_skill("a"), _skill("../evil")], root) + assert report.ok is False + assert isinstance(_read_manifest(root), dict) + + +# --------------------------------------------------------------------------- +# Security abuse matrix +# --------------------------------------------------------------------------- + +HOSTILE_KEYS = [ + "../evil", + "..", + ".", + "", + "/etc/cron.d/x", + "..\\evil", + "c:evil", + "skill:ads", + "sk\0ill", + "-skill", + "Evil", + "a/b", + "x" * 257, + "a/../../b", + "./a", + " leading-space", + "trailing-space ", +] + + +class TestPathTraversal: + """Nothing is ever written outside the root.""" + + @pytest.mark.parametrize("hostile_key", HOSTILE_KEYS) + async def test_hostile_key_is_rejected( + self, tmp_path: Path, hostile_key: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_before = sorted(p.name for p in tmp_path.iterdir()) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(hostile_key)], root) + + assert report.ok is False + assert [a.action for a in report.actions if a.key == hostile_key] == ["error"] + + # The SDK's key validation — not the operating system — must be what + # stopped this. An overlong key exceeds NAME_MAX, a null byte raises in + # the path API, and an absolute path outside the root usually fails on + # permissions, so "an error was reported" is not evidence of a defense + # (and the absolute-path verdict would flip on a privileged runner). + # Assert instead that no write was ever attempted. + assert spy.calls == [] + + # Nothing created outside the root, and no skill directory inside it. + assert sorted(p.name for p in tmp_path.iterdir()) == outside_before + assert [p.name for p in root.iterdir() if p.name != MANIFEST_NAME] == [] + assert list(root.rglob("SKILL.md")) == [] + + async def test_interception_hook_fires_for_a_valid_key( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Positive control for the ``spy.calls == []`` assertion above.""" + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("ok-key")], root) + + assert report.ok is True + assert [Path(call.dst).name for call in spy.calls] == ["SKILL.md"] + + async def test_long_but_filesystem_legal_key_is_written(self, root: Path) -> None: + """The ≤ 256 length bound cannot be exercised through ``write_skills``. + + A key becomes a single directory name and NAME_MAX is 255 bytes on Linux + and macOS, so the longest key the data model permits cannot exist on + disk at all. Assert the accepting side at the largest writable length; + the bound itself is covered by the pure layers (config validation and + accessor revalidation). + """ + key = "k" * 255 + report = await write_skills([_skill(key)], root) + + assert report.ok is True + assert (root / key / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_key_at_the_data_model_bound_is_reported_not_raised( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A 256-character key is valid to every pure layer but fits no filesystem. + + Config validation and the accessors must both accept exactly 256 + characters, yet NAME_MAX is 255 + on Linux and macOS, so this key reaches ``write_skills`` legitimately and + cannot become a directory. Every outcome must be visible in + the report, so it must surface as an ``error`` action rather than an + ``OSError`` escaping the call — which would also skip the manifest rewrite + and orphan any file already written in the same run. + """ + spy = _ReplaceSpy().install(monkeypatch) + long_key = "a" * 256 + + report = await write_skills([_skill("good"), _skill(long_key)], root) + + by_key = _actions_by_key(report) + assert by_key[long_key].action == "error" + assert by_key["good"].action == "written" + # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no + # directory, so "the path does not contain the hostile key" is no longer + # a meaningful check. Assert the stronger thing instead: the only rename + # that happened was into the valid skill's own directory. + assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # The valid skill is fully reconciled: written AND recorded, not orphaned. + assert (root / "good" / "SKILL.md").exists() + assert "good/SKILL.md" in _read_manifest(root)["entries"] + + async def test_valid_keys_still_write_alongside_rejected_ones( + self, root: Path + ) -> None: + report = await write_skills([_skill("good"), _skill("../evil")], root) + by_key = _actions_by_key(report) + assert by_key["good"].action == "written" + assert by_key["../evil"].action == "error" + assert (root / "good" / "SKILL.md").exists() + + async def test_traversal_key_does_not_create_parent_files( + self, tmp_path: Path + ) -> None: + root = tmp_path / "skills" + root.mkdir() + await write_skills([_skill("../../escaped")], root) + assert not (tmp_path / "escaped").exists() + assert not (tmp_path.parent / "escaped").exists() + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestSymlinkAttacks: + """Never write through a symlink.""" + + async def test_symlinked_root_raises(self, tmp_path: Path) -> None: + real_dir = tmp_path / "real" + real_dir.mkdir() + link_root = tmp_path / "link" + link_root.symlink_to(real_dir, target_is_directory=True) + + with pytest.raises(ValueError): + await write_skills([_skill("a")], link_root) + + assert list(real_dir.iterdir()) == [] + + async def test_symlinked_skill_directory_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert list(outside.iterdir()) == [] + + async def test_symlinked_target_file_is_refused(self, tmp_path: Path) -> None: + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + (root / "a" / "SKILL.md").symlink_to(outside_file) + # Manifest lists the path so clobber protection is not what saves us. + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([_skill("a", 2, "attacker payload\n")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + async def test_symlinked_target_is_not_pruned(self, tmp_path: Path) -> None: + """A manifest-listed path that is a symlink is refused, not unlinked. + + Asserting only that the victim file survives proves nothing here: + unlinking a symlink never touches its target, so that assertion holds + for an implementation with no symlink check at all. The observable + contract is the refusal itself (prune path). + """ + root = tmp_path / "skills" + root.mkdir() + outside_file = tmp_path / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a").mkdir() + link = root / "a" / "SKILL.md" + link.symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert [a for a in report.actions if a.action == "removed"] == [] + # The symlink itself is left in place and stays managed. + assert link.is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert outside_file.read_text(encoding="utf-8") == "victim content\n" + + @_needs_dir_fd + async def test_directory_swapped_at_the_rename_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The swap window is closed, not merely narrowed. + + Every check in the world is worthless if the final rename re-resolves + ``/`` from its path: an attacker holding write permission on + the managed root can replace the validated directory with a symlink in + between and redirect the write out of the root. The rename is therefore + performed relative to a descriptor pinned to the directory that was + checked, so it follows the inode rather than the name. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapDirectoryDuring("replace", root / "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert list(outside.iterdir()) == [] + assert (race.moved_to / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + + @_needs_dir_fd + async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The same window on the destructive side. + + ``unlink`` never follows a *trailing* symlink, but it does resolve the + directory above it, so the swap turns a prune into a delete of an + attacker-chosen outside file. The unlink is descriptor-relative for the + same reason the rename is. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + victim = outside / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapDirectoryDuring("unlink", root / "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (race.moved_to / "SKILL.md").exists() + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + + +class TestWithoutDirFd: + """The full-path fallback for platforms with no ``*at()`` family. + + On Windows ``os.open`` cannot open a directory at all, so acquiring the + descriptor must not even be attempted there — a fallback reached only after + a descriptor open would leave every write, prune and manifest rewrite + failing rather than falling back. These tests force the flag off so the + fallback is exercised on POSIX too. + """ + + @pytest.fixture(autouse=True) + def _no_dir_fd(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Models Windows: no ``*at()`` family, and directories cannot be opened. + + Forcing the flag off alone would not reproduce the platform, because + ``os.open`` on a directory succeeds on POSIX — the fallback would be + reached either way. Making that call raise the ``PermissionError`` + Windows raises is what proves the descriptor open is never attempted. + """ + monkeypatch.setattr(safe_fs_module, "SUPPORTS_DIR_FD", False) + real_open = os.open + + def no_directory_open(path: Any, *args: Any, **kwargs: Any) -> int: + if os.path.isdir(path): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(safe_fs_module.os, "open", no_directory_open) + + async def test_write_prune_and_manifest_all_succeed(self, root: Path) -> None: + first = await write_skills([_skill("a"), _skill("b")], root) + assert first.ok is True, _error_messages(first) + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + assert _manifest_path(root).exists() + assert stat.S_IMODE((root / "a" / "SKILL.md").stat().st_mode) == 0o644 + + second = await write_skills([_skill("a")], root) + + assert second.ok is True, _error_messages(second) + assert not (root / "b" / "SKILL.md").exists() + assert "b/SKILL.md" not in _read_manifest(root)["entries"] + + async def test_a_symlinked_skill_directory_is_still_refused( + self, root: Path, tmp_path: Path + ) -> None: + """The fallback keeps the ``lstat`` floor: no writing through a link.""" + outside = tmp_path / "outside" + outside.mkdir() + (root / "a").symlink_to(outside, target_is_directory=True) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert list(outside.iterdir()) == [] + + +class TestNonRegularFiles: + """A managed path that is not a regular file is refused, never read.""" + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_a_fifo_at_the_managed_path_does_not_block(self, root: Path) -> None: + """Reading a FIFO with no writer blocks forever. + + Same attacker capability the symlink checks defend against: swapping a + managed ``SKILL.md`` for a FIFO would otherwise hang the whole reconcile + — and the caller's event loop with it — well past any ``timeout``, since + the deadline is only consulted between steps. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + +class TestClobberProtection: + """Destructive ops only on manifest-listed paths.""" + + async def test_unmanaged_file_is_never_overwritten(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_unmanaged_file_is_never_deleted(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + + await write_skills([], root) + + assert target.read_text(encoding="utf-8") == "user authored\n" + + async def test_manifest_entry_with_mismatched_key_does_not_authorize( + self, root: Path + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text("user authored\n", encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "a/SKILL.md": _entry("different-key", 1, "user authored\n") + }, + }, + ) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == "user authored\n" + + +DIVERGENT_CONTENT = "existing content\n" + + +def _live_entries() -> dict[str, Any]: + """A parseable entries map that really does claim ``a/SKILL.md`` as managed.""" + return {"a/SKILL.md": _entry("a", 1, DIVERGENT_CONTENT)} + + +# The first six variants are unparseable: ``entries`` is missing, the wrong type, +# or the whole document is garbage. That makes "performed no destructive action" +# arithmetic rather than a defense — with no entries to act on, a file at a +# managed path is protected by clobber protection and there is nothing to prune, +# so those cases pass against an implementation that simply treats a corrupt +# manifest as an empty one. +# +# The ``*_live_entries`` variants are the ones that actually test round-tripping: corrupt +# ONLY in ``manifestVersion``, with a valid entries map listing the managed path +# under a matching key. The implementation has everything it needs to overwrite +# and to prune, and must refuse anyway. +CORRUPT_MANIFESTS: list[tuple[str, Any]] = [ + ("garbage", "{not json at all"), + ("empty", ""), + ("wrong_types", {"manifestVersion": 1, "entries": ["a/SKILL.md"]}), + ("entries_missing", {"manifestVersion": 1}), + ("future_version", {"manifestVersion": 2, "entries": {}}), + ("version_not_int", {"manifestVersion": "1", "entries": {}}), + ("future_version_live_entries", {"manifestVersion": 2, "entries": _live_entries()}), + ( + "version_not_int_live_entries", + {"manifestVersion": "1", "entries": _live_entries()}, + ), +] + +LIVE_ENTRY_MANIFESTS: list[tuple[str, Any]] = [ + case for case in CORRUPT_MANIFESTS if case[0].endswith("_live_entries") +] + + +class TestCorruptManifest: + """Corrupt manifest fails closed, non-destructively.""" + + @pytest.mark.parametrize( + "raw", + [case[1] for case in CORRUPT_MANIFESTS], + ids=[case[0] for case in CORRUPT_MANIFESTS], + ) + async def test_no_destructive_action_and_error_reported( + self, root: Path, raw: Any + ) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([_skill("a", 2, "new content\n")], root) + + assert report.ok is False + # The error must name the manifest. For the unparseable variants the file + # at the managed path is also unmanaged, so a bare "some error happened" + # assertion is satisfied by clobber protection alone and says nothing + # about whether the manifest state was detected at all. + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + + async def test_run_level_error_carries_the_empty_key_sentinel( + self, root: Path + ) -> None: + """A run-level error has no skill key to hang off. + + The empty string is public API surface: a caller grouping the report by + key has to know the sentinel exists. Asserted here rather than in the + parametrized cases above so it is a statement about the manifest error + specifically, not about whichever error happens to come first. + """ + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("a")], root) + + manifest_errors = [ + action + for action in report.errors + if "manifest" in (action.error or "").lower() + ] + assert manifest_errors, _error_messages(report) + assert all(action.key == "" for action in manifest_errors) + # A per-skill error in the same report still carries its real key, so the + # sentinel is not simply "every error action has an empty key". + assert all( + action.key != "" + for action in report.errors + if action not in manifest_errors + ) + + @pytest.mark.parametrize( + "raw", + [case[1] for case in LIVE_ENTRY_MANIFESTS], + ids=[case[0] for case in LIVE_ENTRY_MANIFESTS], + ) + async def test_managed_file_is_not_pruned_when_only_the_version_is_corrupt( + self, root: Path, raw: Any + ) -> None: + """The prune counterpart of the live-entries cases. + + Here the implementation can read the entries map and knows exactly which + file it owns, so refusing to remove it is a real decision rather than an + absence of information. + """ + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, raw) + + report = await write_skills([], root) + + assert report.ok is False + assert target.read_text(encoding="utf-8") == DIVERGENT_CONTENT + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_nothing_is_pruned_under_a_corrupt_manifest(self, root: Path) -> None: + target = root / "a" / "SKILL.md" + target.parent.mkdir() + target.write_text(DIVERGENT_CONTENT, encoding="utf-8") + _write_manifest(root, "{not json at all") + + report = await write_skills([], root) + + assert report.ok is False + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + errors = _error_messages(report) + assert any("manifest" in e.lower() for e in errors), errors + + async def test_brand_new_paths_may_still_be_written(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + report = await write_skills([_skill("fresh")], root) + + actions = _actions_by_key(report) + assert actions["fresh"].action == "written" + assert (root / "fresh" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: + _write_manifest(root, "{not json at all") + + await write_skills([], root) + + assert _manifest_path(root).exists() + assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" + + +class TestWriteSkillsTelemetry: + """Materialized / revoked signals from write_skills.""" + + async def test_materialized_signal_per_action( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "same", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + }, + }, + ) + (root / "stale").mkdir() + (root / "stale" / "SKILL.md").write_text("old\n", encoding="utf-8") + + await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + signals = recording_emitter.signals(MATERIALIZED_SIGNAL) + by_key = {s["skill_key"]: s for s in signals} + assert len(signals) == 3 + assert by_key["same"]["reconcile_action"] == "skipped_current" + assert by_key["stale"]["reconcile_action"] == "updated" + assert by_key["brand-new"]["reconcile_action"] == "written" + + async def test_materialized_signal_properties( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["skill_key"] == "a" + assert props["content_bytes"] == len(SKILL_BODY.encode("utf-8")) + assert props["content_hash"] == _hash(SKILL_BODY) + assert props["reconcile_action"] == "written" + assert props["language"] == "python" + + async def test_no_filesystem_paths_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + assert "target_path" not in props + for value in props.values(): + assert str(root) not in str(value) + + async def test_no_skill_body_in_telemetry( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + + await write_skills([_skill("a")], root) + + for _signal, props in recording_emitter.records: + for value in props.values(): + assert "Do the thing." not in str(value) + + async def test_revoked_signal_on_prune( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY, version=4) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert revoked[0]["skill_key"] == "gone" + assert revoked[0]["version"] == 4 + assert revoked[0]["removed_from_disk"] is True + assert revoked[0]["language"] == "python" + + async def test_revoked_signal_redacts_an_untrusted_manifest_version( + self, root: Path, recording_emitter: Any + ) -> None: + """The manifest is untrusted, so its version is shape-checked first. + + Anything with write access to the managed root can plant an arbitrary + string here; echoing it verbatim would publish attacker-controlled + content — a skill body, or PII — as a signal property. + """ + skills_module._set_emitter_for_testing(recording_emitter) + target = root / "gone" / "SKILL.md" + target.parent.mkdir(parents=True) + target.write_text(SKILL_BODY, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "gone/SKILL.md": { + "key": "gone", + "version": "Do the thing. " * 8, + "sha256": _hash(SKILL_BODY), + } + }, + }, + ) + + await write_skills([], root) + + revoked = recording_emitter.signals(REVOKED_SIGNAL) + assert len(revoked) == 1 + assert "version" not in revoked[0] + assert revoked[0]["skill_key"] == "gone" + for value in revoked[0].values(): + assert "Do the thing." not in str(value) + + async def test_no_revoked_signal_when_prune_disabled( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + _place_managed(root, "gone", SKILL_BODY) + + await write_skills([], root, prune=False) + + assert recording_emitter.signals(REVOKED_SIGNAL) == [] + + async def test_write_skills_records_no_signal_outside_the_approved_set( + self, root: Path, recording_emitter: Any + ) -> None: + """Allowlist sweep over a run that exercises all four actions. + + The accessor-side half of this sweep is + ``test_accessors_record_no_signal_outside_the_approved_set`` in + test_skills.py. Asserted over recorded strings, so no module-level + signal-name constant is required of the implementation. + """ + skills_module._set_emitter_for_testing(recording_emitter) + for key, content in (("same", SKILL_BODY), ("stale", "old\n"), ("gone", "g\n")): + (root / key).mkdir() + (root / key / "SKILL.md").write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + "same/SKILL.md": _entry("same", 1, SKILL_BODY), + "stale/SKILL.md": _entry("stale", 1, "old\n"), + "gone/SKILL.md": _entry("gone", 1, "g\n"), + }, + }, + ) + + report = await write_skills( + [_skill("same"), _skill("stale", 2, "fresh\n"), _skill("brand-new")], + root, + ) + + # Positive control: the subset assertion is vacuous unless the run + # really did produce all four actions and record for them. + assert {a.action for a in report.actions} == { + "skipped_current", + "updated", + "written", + "removed", + } + recorded = {signal for signal, _props in recording_emitter.records} + assert recorded <= APPROVED_SIGNALS, ( + f"unapproved signal(s): {sorted(recorded - APPROVED_SIGNALS)}" + ) + assert not recorded & REMOVED_SIGNALS + assert recorded == {MATERIALIZED_SIGNAL, REVOKED_SIGNAL} + + async def test_no_ld_track_calls_from_write_skills( + self, root: Path, mock_ld_client: Any + ) -> None: + await init_client(client=mock_ld_client) + + await write_skills([_skill("a"), _skill("../evil")], root) + + mock_ld_client.track.assert_not_called() + + async def test_throwing_emitter_never_breaks_the_reconcile( + self, root: Path, throwing_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(throwing_emitter) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_integrity_signal_property_keys_match_across_layers( + self, root: Path, recording_emitter: Any + ) -> None: + """The same defect, caught at either layer, records + the same property keys. + + Verification runs twice by design: once at the accessor boundary and + again immediately before a write. The signal contract marks ``expected_hash`` + optional, so an implementation that populates it on one path and omits + it on the other passes every other assertion here while making the + signal's shape depend on which internal code path noticed. Oversize + content is the case reachable from both layers with the expected hash in + hand throughout. + """ + skills_module._set_emitter_for_testing(recording_emitter) + oversize = "x" * (64 * 1024 + 1) + content_hash = _hash(oversize) + + # Layer 1 — the accessor boundary. + store = InMemorySkillStore() + store.put( + { + "key": "big", + "version": 1, + "content": oversize, + "contentHash": content_hash, + } + ) + skills_module._set_store(store) + assert await get_skill("big") is None + + # Layer 2 — verify-then-write, on a directly constructed Skill. + report = await write_skills( + [ + Skill( + key="big", + version=1, + content=oversize.encode("utf-8"), + content_hash=content_hash, + ) + ], + root, + ) + assert report.ok is False + + failures = recording_emitter.signals(INTEGRITY_SIGNAL) + assert len(failures) == 2, failures + accessor_keys, write_keys = (set(props) for props in failures) + assert accessor_keys == write_keys, ( + f"accessor-only keys: {sorted(accessor_keys - write_keys)}; " + f"write-only keys: {sorted(write_keys - accessor_keys)}" + ) + assert "expected_hash" in accessor_keys From 568729d3945d1448f4f1a17d066155ad4c269970 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 25 Aug 2026 16:10:53 -0400 Subject: [PATCH 06/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20materialize=20onto=20disk=20under=20a=20manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth of five slices. Adds `write_skills`, which writes `//SKILL.md` and reconciles against a manifest recording what the SDK owns, so it overwrites or removes only files it wrote — a file you placed yourself is reported and left untouched. report = await write_skills(refs, ".claude/skills") `skills` accepts `Skill` values, references, bare keys, or the literal `"*"` for everything the store holds. Every outcome is visible in the returned `ReconcileReport`: one `ReconcileAction` per skill, carrying `written`, `updated`, `skipped_current`, `removed` or `error`, plus `.ok` and `.errors`. A failure belonging to the run rather than to one skill — an unreadable manifest, a retrieval that failed before any key was known — carries the empty string as its key. Writes are atomic, at mode 0644, and every destructive step runs against a descriptor pinned to a directory that was already checked, so a path swapped after the check cannot redirect a write or an unlink out of the managed root. Where the platform has no `*at()` family the identical sequence runs against full paths. The defenses, all of them deliberate and all of them tested: - The key is re-validated here regardless of upstream validation, before any filesystem call, because a key becomes a directory name. The data model allows 256 characters and `NAME_MAX` is 255 bytes, so an over-long key is refused too. - Never write or unlink through a symlink, on the write path or the prune path. - Destruction only on manifest-listed paths whose key matches. - A corrupt manifest fails closed: no overwrites and no prunes, brand-new paths may still be written, an error action names the manifest, and the manifest is not rewritten. - An incomplete retrieval suppresses pruning, so a transport outage cannot read as "everything was revoked". - Content is re-verified immediately before the write, because a `Skill` can also be constructed directly by a caller. Pruning removes formerly-managed skills that are no longer referenced, which is how revocation takes effect. `timeout` bounds retrieval, the writes and the pruning; only the final manifest rewrite runs past it, so files already written are never orphaned. `write_skills` performs synchronous filesystem I/O and does not yield — it is `async` for signature parity with the other accessors. Reconcile one root at a time: a run is atomic against the rest of the loop today, so wrapping it to run concurrently makes two runs against one root race on the manifest. The `"*"` form collapses to one object per key at its newest version, since `//SKILL.md` is a single path and writing it twice in one run is a bug rather than a policy, and it reports a withholding count at WARN for the same reason the accessors do. The security abuse matrix — path traversal, symlink attacks, clobber protection, corrupt manifests, atomicity under an injected crash, and the materialization telemetry allowlist — is the next slice. The guards it exercises are all here; what lands next is the adversary that proves each one fails without them. Testing: `uv run pytest` → 1216 passed. `ruff check`, `ruff format --check`, and `mypy packages/*/src` all clean. Co-Authored-By: Claude Fable 5 --- packages/client/README.md | 63 +- packages/client/agents.md | 114 ++- .../src/launchdarkly_ai_server/__init__.py | 21 + .../src/launchdarkly_ai_server/skills.py | 4 +- .../src/launchdarkly_ai_server/skills_fs.py | 960 ++++++++++++++++++ .../src/launchdarkly_ai_server/types.py | 48 + packages/client/tests/test_skills.py | 123 ++- packages/client/tests/test_skills_fs.py | 582 +++++++++++ 8 files changed, 1889 insertions(+), 26 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fs.py create mode 100644 packages/client/tests/test_skills_fs.py diff --git a/packages/client/README.md b/packages/client/README.md index ae572b3b..d9c1db15 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -223,15 +223,17 @@ asyncio.run(main()) ### Agent Skills Skills are versioned `SKILL.md` documents managed in LaunchDarkly and attached to AI Config -variations by reference. The SDK surfaces which skills a config references and retrieves -their content. Materializing them onto disk, where agent runtimes discover them, follows. +variations by reference. The SDK surfaces which skills a config references, retrieves their +content, and materializes them onto disk where agent runtimes (Claude Agent SDK, and +anything else following the `//SKILL.md` convention) discover them. ```python import asyncio import hashlib +from pathlib import Path from launchdarkly_ai_server import ( - init_client, inspect_config, skill_refs, get_skill, get_skills, + init_client, inspect_config, skill_refs, get_skill, write_skills, InMemorySkillStore, ) @@ -260,13 +262,18 @@ async def main(): if skill is not None: print(skill.content) - # 3. Or resolve the config's references in one call. - for s in await get_skills(refs): - print(s.key, s.version) + # 3. Write them where the agent runtime will look. Only the leaf directory is + # created, so the parent must already exist. + Path(".claude").mkdir(exist_ok=True) + report = await write_skills(refs, ".claude/skills") + for action in report.errors: + print(f"skill {action.key or ''}: {action.error}") asyncio.run(main()) ``` +Pass `"*"` instead of a reference list to materialize every skill the store holds. + **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an integer ≥ 1): the whole variation is rejected, `inspect_config` returns `config: None`, and @@ -333,7 +340,22 @@ treat `expected_hash` / `observed_hash` as the evidence pair. **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version -1 and gets it even when a newer one is also held. +1 and gets it even when a newer one is also held. `all_skills()` and `write_skills("*")` +collapse to one skill per key at its newest version, since `//SKILL.md` is a +single path. + +**The root's parent must exist.** `write_skills` creates the root itself but never its +ancestors, so a typo cannot scatter a directory tree across your project. An absent parent, +a root that is an existing file, and a root that is a symlink each raise `ValueError` — +these are caller errors, distinct from the per-skill `error` actions in the report. + +**`write_skills` is deliberately conservative** about your filesystem. It writes only +`//SKILL.md`, tracks what it owns in a manifest at +`/.launchdarkly-skills.json`, and will overwrite or delete **only** paths that +manifest records. A file you placed yourself is reported as an error and left untouched; it +never writes through a symlink; writes are atomic (temp file, `fsync`, rename) at mode +`0644`; and if the manifest is unreadable it performs no destructive action at all. Removing +a skill from a variation is how revocation works — the next reconcile prunes it. | Export | Description | |---|---| @@ -341,11 +363,34 @@ version a variation currently pins. `get_skill("k", version=1)` asks the store f | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | +| `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, -the accessors raise `RuntimeError` explaining what to do. `shutdown()` clears it. +the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the +failure in its report (or raises, with `on_unavailable="raise"`). `shutdown()` clears it. + +`ReconcileReport.actions` holds one `ReconcileAction` per outcome — `written`, `updated`, +`skipped_current`, `removed`, or `error` — each carrying `key`, `version`, the resolved +`path`, and `error`. `report.ok` is `True` when no action is an `error`, and +`report.errors` is just the `error` actions, so you rarely need to filter `actions` +yourself. A failure that belongs to the whole run rather than to one skill — an unreadable +manifest, for instance — carries the empty string as its `key`. + +The fixed on-disk values are exported too, so you do not have to hardcode them: +`MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / +`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). + +**`write_skills` blocks.** It is `async` for parity with the other accessors and with the +TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, +so a large reconcile holds the event loop for its duration. Wrap it in +`asyncio.to_thread` if that matters. For the same reason `timeout` is checked between +steps rather than interrupting one already in progress. Reconcile one root at a time, +though: because nothing yields today, a run is atomic against the rest of your loop, and +wrapping it to run concurrently makes two runs against the same root race on the manifest. `all_objects` returns one entry per `(key, version)` under keys that are **opaque** to the SDK — identity is read from each object's own `key` and `version` fields, so a store is free @@ -389,3 +434,5 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | +| `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index 152f5771..feb2e348 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | @@ -56,7 +57,7 @@ from launchdarkly_ai_server import ( TrackData, UsageDict, HandlerResult, HandlerStreamEvent, StreamEvent, StreamChunkEvent, StreamDoneEvent, ExecuteStreamEvent, ExecuteStreamDoneEvent, VariationMeta, InitClientOptions, JudgeResult, ParseResult, ParseSuccess, ParseFailure, - Skill, SkillReference, + Skill, SkillReference, ReconcileAction, ReconcileReport, ) # Utilities @@ -76,8 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, + skill_refs, get_skill, get_skills, all_skills, write_skills, SkillStore, InMemorySkillStore, + SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, + ReconcileActionKind, OnUnavailable, # the two closed-set unions ) ``` @@ -179,7 +182,7 @@ This is an OTel context value, not W3C baggage, so the id does not leak onto out Versioned `SKILL.md` documents attached to AI Config variations by reference, retrieved through an injectable store, and materialized onto disk for agent runtimes to discover. -Three layers, in increasing order of blast radius. Only the first is implemented here: +Three layers, in increasing order of blast radius: 1. **Reference discovery** — `skill_refs(config)` projects the config's `skills` array into typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. @@ -190,7 +193,8 @@ Three layers, in increasing order of blast radius. Only the first is implemented `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. -3. **Materialization** — writing skills onto disk under a manifest. +3. **Materialization** — `write_skills(skills, root)` writes `//SKILL.md` and + reconciles against a manifest at `/.launchdarkly-skills.json`. ### The store seam, and why version is part of the lookup @@ -241,6 +245,30 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Attacker-controlled strings are never echoed into telemetry.** `contentHash` and `key` come off the wire, so a store could put the skill body in either; both are shape-checked and redacted before they reach a signal or a log line. +- **The key is re-validated inside `write_skills`**, regardless of upstream validation — a + key becomes a directory name. Rejection happens before any filesystem call. +- **Never write through a symlink**, in either the skill directory or the target file, on + the write path *and* the prune path. +- **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a + managed path with no matching manifest entry is reported as `error` and left alone. +- **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed + `entries`, or a `manifestVersion` this release cannot read means no overwrites and no + prunes, brand-new paths may still be written, an `error` action names the manifest, and + the manifest file itself is not rewritten. +- **An incomplete retrieval suppresses pruning.** Otherwise a transport outage would read + as "everything was revoked" and delete the customer's managed files. +- **Writes are atomic**: temp file created exclusively in the target's *own* directory, + mode `0644` set explicitly (never inherited from the umask, never executable), write, + fsync, `os.replace`, fsync the directory. `os.replace` is the single rename call site + and must not be swapped for `os.rename`. +- **Every operation under the root goes through a pinned descriptor, not a path.** See + "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at + write or unlink time reopens a swap window that the checks above cannot cover. +- **A key valid to the data model may still be unrepresentable on disk.** The model allows + 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before + any filesystem call, and every per-skill filesystem failure is caught at the loop so it + becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan + files already written in that run. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently @@ -262,9 +290,6 @@ Exactly three signals exist, and the list is an **allowlist, not a floor**: | `AgentControl Skill Materialized` | each `written` / `updated` / `skipped_current` | `skill_key`, `content_bytes`, `content_hash`, `reconcile_action`, `language` | | `AgentControl Skill Revoked Received` | prune removes a formerly managed skill | `skill_key`, `version`, `removed_from_disk`, `language` | -The last two belong to the materialization layer and have no caller yet; they live here -with the first so the allowlist is one section of one file rather than three sites to audit. - ### The integrity-failure log record The signal above is product telemetry; the **log record** beside it is the customer-owned @@ -321,11 +346,12 @@ where they cannot see it. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and **deliberately excluded from SDK emission** — both are observable server-side. Do not add them. The skill body never appears in a signal, a log line, or -an error message, and no signal carries a filesystem path. An emitter that raises is caught -and logged; it never fails the operation. +an error message, and no signal carries a filesystem path (paths belong in the returned +`ReconcileReport`, which is user-facing API). An emitter that raises is caught and logged; +it never fails the operation. -Module state lives in `skills_core.py`, so there is exactly one store and one emitter -however the feature is entered. All three signals are emitted from the `record_*` functions +Module state lives in `skills_core.py`, the module `skills.py` and `skills_fs.py` share, +so there is exactly one store and one emitter however the feature is entered. All three signals are emitted from the `record_*` functions next to the seam there — nothing outside that module calls `emit`, so the allowlist is enforced in one place. @@ -366,7 +392,14 @@ primitives live in `safe_fs.py`, which knows nothing about skills: *trailing* symlink, but it does resolve the directory above it, so the same swap turns a removal into a delete of an attacker-chosen file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is - not what the caller believes, and that is the caller's to report. + not what the caller believes, and that is the caller's to report. `_prune_one` goes + through it; `rmdir` stays path-based and is safe that way, since it fails `ENOTDIR` on a + symlink and only ever succeeds on an empty directory. + +Every `lstat`, `realpath` and containment check on the skills side lives in one shared +`_unsafe_path_reason`, so the write and prune paths cannot drift apart on what counts as +unsafe. `skills_fs._prune_one` spells its symlink check `os.stat(..., follow_symlinks=False)` +rather than `os.lstat`, matching the name the capability probe advertises. `safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` @@ -379,7 +412,43 @@ and silently turns the defense off, so the probe names the advertised twins (Windows) `open_directory_nofollow` returns `None` after an `lstat` check instead of attempting the descriptor open — `os.open` cannot open a directory there — and every caller falls back to the identical full-path sequence, the per-component `lstat` floor. The -residual window on those platforms is documented rather than closed. +residual window on those platforms is documented rather than closed; the TOCTOU tests skip +off this same flag, deliberately, so a probe that wrongly reports "unsupported" cannot also +silently skip the tests that would have caught it. + +Both call shapes are admitted by the test seam. `os.replace` remains the single +interceptable rename call site; under the descriptor-relative shape `dst` is the bare string +`"SKILL.md"`, so an `endswith("SKILL.md")` spy filter still matches, and the +same-directory requirement is proved by descriptor identity (`src_dir_fd == dst_dir_fd`, +resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing path strings. +A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes +it as soon as the write returns. + +### Deferred: bounded retries + +`timeout` is implemented — a monotonic deadline, checked before each retrieval, before +each write, and before each prune; only the final manifest rewrite runs past it, so files +already written are never orphaned. Bounded retries inside that deadline are **not** +implemented, and belong to the delivery transport, not to this layer. Three structural +reasons, all of which the transport changes: + +1. **There is nothing transient to retry.** `SkillStore.get_object` is a synchronous + in-process read against already-delivered data, modelled on the LaunchDarkly + data-store API. `InMemorySkillStore` reads a dict. A retry re-invokes customer code and + returns the same answer. +2. **The seam cannot classify a failure.** All it surfaces is "this raised". Retrying a + `PermissionError` or a malformed payload spends the caller's `timeout` on a certainty. + The transient/permanent taxonomy a retry policy needs is the transport's to define. +3. **Backoff has nowhere to sleep.** The retrieval path (`_resolve_requests`, + `_resolve_reference`, `_resolve_all`) is synchronous, called from an async + `write_skills`. Backoff would mean either `time.sleep` — blocking the event loop of every + caller — or async-ifying the whole path for a store that cannot benefit. + +Picking a bound and a backoff now would fix numbers in a cross-language contract with no +transport to calibrate them against, so there is **no** retry test and no assumable attempt +count. When the transport lands it owns the policy; keep both languages retry-free until +then, since the number of times a throwing store is invoked is observable and the two would +otherwise diverge. --- @@ -491,7 +560,7 @@ Install with `pip install "launchdarkly-ai-server[otel]"`; see [OTel Setup](#ote |---|---| | `launchdarkly-server-sdk>=9.0`, and the `otel` extra mirrored (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) | Each dynamically-resolved or optional package is repeated in the dev group so the test suite can import it. Something that is *only* optional would not be installed in this workspace and the tests covering its present-and-working path could not run. | | `pytest>=8`, `pytest-asyncio>=0.24` | Test runner and the async support the whole suite relies on. `asyncio_mode = "auto"` is set at the workspace root, which is why no test in this package carries an `@pytest.mark.asyncio`. | -| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. | +| `mypy>=1.10` (`strict`), `ruff>=0.15` | Type checker and linter/formatter. `mypy` strict mode is the only thing enforcing the `Literal[...]` closed set on `ReconcileAction.action` — unlike `write_skills`'s `on_unavailable`, which is also checked at runtime because the value can arrive from untyped code. | --- @@ -513,6 +582,22 @@ convenience accessor that reads meaning into it — no YAML/frontmatter parsing, verified verbatim byte buffer and nothing more; a consumer who wants structure parses it on their side of the boundary. +### 4. Assuming `write_skills` prunes on every run + +Pruning is suppressed when the manifest is corrupt or any retrieval was incomplete — both +mean the SDK cannot tell what it owns or what is still current, and deleting under that +uncertainty is data loss. A run whose report contains a manifest `error` will not have +pruned anything, so do not read "no `removed` actions" as "nothing is stale". + +### 5. Treating "absent from the resolved set" as always meaning revoked + +Revocation is pruning, but only for a skill the store genuinely no longer serves. An object +that is *present and unverifiable* is a different thing, and `_resolve_all` must emit a +failed `_PendingWrite` for it rather than filtering it out: dropping it silently leaves its +key out of the requested set, so prune deletes the last known-good copy on disk and reports +a routine `removed` with `report.ok` still true. Tampered content must never be able to +trigger deletion. + --- ## Adding a New Export @@ -532,4 +617,5 @@ on their side of the boundary. - Do not route skills telemetry through `client.track()`, and do not introduce an LD context anywhere in the skills path. Signals go through the `skills_core.py` emitter seam, whose default is a no-op, and only via its `record_*` functions. - Do not add a signal name outside the three in the Agent Skills table above — the list is an allowlist. `AgentControl Skill SDK Reference Returned` and `AgentControl Skill Content Retrieved` were considered and deliberately excluded from SDK emission. - Do not rename `ld.skills.integrity_failure`, and do not add a ninth `reason_code` in one language only — both are documented compatibility surfaces. See "The integrity-failure log record" above. +- Do not relax any of the `write_skills` filesystem defenses (local key re-validation, symlink refusal, manifest-authorized destruction, corrupt-manifest fail-closed, atomic `0644` writes). Each is a deliberate security property with abuse-case tests attached. - Do not make `SkillStore` lookups key-only. Version is part of the lookup identity because a payload holds several versions of one key; a key-only seam cannot express a version-pinned reference. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 02c858f2..37b1570a 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -45,6 +45,13 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fs import ( + MANIFEST_FILENAME, + MANIFEST_VERSION, + SKILL_FILENAME, + OnUnavailable, + write_skills, +) from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -74,6 +81,9 @@ ProviderGraphResponse, ProviderHandler, ProviderResponse, + ReconcileAction, + ReconcileActionKind, + ReconcileReport, Skill, SkillReference, StreamChunkEvent, @@ -136,6 +146,9 @@ "ProviderGraphResponse", "ProviderHandler", "ProviderResponse", + "ReconcileAction", + "ReconcileActionKind", + "ReconcileReport", "Skill", "SkillReference", "StreamChunkEvent", @@ -216,6 +229,14 @@ "get_skill", "get_skills", "all_skills", + "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the two closed-set unions a typed consumer needs to name + "ReconcileActionKind", + "OnUnavailable", + # skills — on-disk constants, identical across languages + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", ] diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index eb3946c2..9d9b3627 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -183,8 +183,8 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: A config that came through ``parse_ai_config`` never contains an invalid entry — parsing fails closed on one. A hand-built dict can, and a silently - shortened projection would leave a caller materializing a skill set it - believes is complete, so every dropped entry is logged. + shortened projection would let ``write_skills`` prune the dropped skill's + on-disk copy, so every dropped entry is logged. """ if not isinstance(config, dict): return [] diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py new file mode 100644 index 00000000..feb8fbc9 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -0,0 +1,960 @@ +""" +Agent Skills — filesystem materialization. + +The highest-blast-radius layer of the feature: this is the part that writes to a +customer's disk. Split out of ``skills.py`` on that boundary — everything here +takes already-verified content and reconciles it against a managed root, while +``skills.py`` owns retrieval and verification and knows nothing about the +filesystem. The dependency runs one way only, and the descriptor-pinned +primitives every destructive step goes through live in ``safe_fs.py``. + +The reconcile is manifest-driven and fails closed: destructive operations only +ever touch paths ``/.launchdarkly-skills.json`` records under a matching +key, a corrupt manifest suppresses every destructive action, and an incomplete +retrieval suppresses pruning. Content is re-verified immediately before the +write, because a ``Skill`` can also be constructed directly by a caller. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import stat +import time +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +from .safe_fs import ( + SymlinkRefused, + atomic_write, + atomic_write_in, + pinned_directory, + unlink_file, +) +from .skills_core import ( + NO_STORE_MESSAGE, + Resolution, + SkillStore, + VerificationFailure, + get_store, + list_raw_objects, + log_withholding_summary, + newest_by_key, + record_materialized, + record_revoked, + reference_target, + resolve_from_store, + verified_bytes, + verify_raw_skill, +) +from .types import ( + ReconcileAction, + ReconcileActionKind, + ReconcileReport, + Skill, + SkillReference, +) +from .types_validation import ( + is_valid_skill_key, + is_valid_skill_version, + skill_key_rejection_reason, +) + +logger = logging.getLogger(__name__) + +MANIFEST_FILENAME = ".launchdarkly-skills.json" +"""The SDK's record of what it has written under a managed root.""" + +MANIFEST_VERSION = 1 +"""Manifest schema version this release writes, and the highest it can read.""" + +SKILL_FILENAME = "SKILL.md" +"""The single file each skill materializes to, under ``//``.""" + +OnUnavailable = Literal["keep", "raise"] +"""How ``write_skills`` reacts to content it could not retrieve.""" + +_UNAVAILABLE_PREFIX = "skill retrieval unavailable: " +""" +Prefix on every error describing content that could not be retrieved. Callers +assert on it, so it lives in one place. +""" + +_MAX_PATH_COMPONENT_BYTES = 255 +""" +NAME_MAX on Linux and macOS, and the component limit on Windows. A skill key +becomes a single directory name, and the data model permits keys up to 256 +characters — one byte longer than any of those filesystems can represent. Such a +key is rejected before any filesystem call so the caller gets a reported action +rather than an ENAMETOOLONG escaping from a stat deep inside the reconcile. +""" + + +# ------------------------------------------------------------------------- +# The reconcile entry point +# ------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _PendingWrite: + """One skill queued for the reconcile: resolved content, or why there is none.""" + + key: str + skill: Skill | None = None + error: str | None = None + + +async def write_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", +) -> ReconcileReport: + """ + Materializes skills under a managed root at ``//SKILL.md``. + + *skills* is a sequence of ``Skill`` / ``SkillReference`` / key strings, or + the literal ``"*"`` meaning everything ``all_skills()`` returns. ``Skill`` + values are used as-is; references and strings resolve through the accessors, + so they need a configured store. + + The reconcile is manifest-driven (``/.launchdarkly-skills.json``): + destructive operations only ever touch paths the manifest records under a + matching key, so a file the SDK did not write is never overwritten or + deleted. ``prune`` removes formerly-managed skills that are no longer in the + requested set — which is also how revocation takes effect. ``timeout`` + bounds retrieval, the writes, and pruning; the final manifest rewrite + always runs, so files already written are never orphaned. ``on_unavailable`` + chooses between reporting a failed retrieval (``"keep"``, leaving existing + managed files alone) and raising (``"raise"``). + + Returns a ``ReconcileReport`` in which every outcome is visible; raises + ``ValueError`` for a caller error such as an unusable root. + + **This call performs synchronous filesystem I/O and does not yield.** It is + ``async`` for signature parity with the other accessors and with the + TypeScript SDK, not because it awaits anything: every read, write, ``fsync`` + and rename runs inline, so a large reconcile blocks the event loop for its + duration. Wrap it in ``asyncio.to_thread`` if that matters on your loop. + ``timeout`` is checked between steps rather than interrupting one in + progress, for the same reason. + + **One root, one reconcile at a time.** Because nothing here yields, a whole + reconcile is atomic against every other task on the loop today. Wrapping it + to run concurrently makes that the caller's problem instead: two runs + against the same root interleave on the manifest, and the loser's entries + are lost — which leaves the files it wrote unmanaged, and a later reconcile + then refuses them as files the SDK did not write. + """ + # Both of these are annotated as closed sets, but the values can still arrive + # from untyped code, so they are checked rather than assumed. + if on_unavailable not in ("keep", "raise"): + raise ValueError( + f'on_unavailable must be "keep" or "raise", got {on_unavailable!r}' + ) + if timeout < 0: + raise ValueError(f"timeout must not be negative, got {timeout!r}") + + deadline = time.monotonic() + timeout + root_path = _resolve_root(root) + manifest, manifest_error = _load_manifest(root_path) + entries: dict[str, Any] = manifest.get("entries", {}) + + actions: list[ReconcileAction] = [] + if manifest_error is not None: + # Run-level failure: there is no single skill key to hang it off. + actions.append(_run_error(manifest_error)) + + requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) + + written, write_timed_out = _write_all(root_path, requests, entries, deadline) + actions.extend(written) + incomplete = incomplete or write_timed_out + + # Pruning is destructive, so it needs a trustworthy picture of both sides: a + # corrupt manifest means we do not know what we own, and an incomplete run — + # a retrieval that failed, or a deadline that expired mid-write — means we do + # not know what is still current. Either way, deleting would be a guess. + if prune and manifest_error is None and not incomplete: + actions.extend( + _prune( + root_path, + entries, + {request.key for request in requests}, + deadline, + ) + ) + + if manifest_error is None: + actions.extend(_rewrite_manifest(root_path, manifest, entries)) + + return ReconcileReport(actions=actions) + + +_RUN_LEVEL_KEY = "" +""" +The documented sentinel for a failure that belongs to no single skill (see +``ReconcileAction``). Spelled once so every path that cannot attribute a +failure to a key agrees with the others. +""" + + +def _run_error(message: str) -> ReconcileAction: + """ + A failure belonging to the run rather than to one skill. + + Uses the run-level sentinel key; it is constructed here so every run-level + error agrees. + """ + return ReconcileAction(key=_RUN_LEVEL_KEY, action="error", error=message) + + +def _write_all( + root: Path, + requests: list[_PendingWrite], + entries: dict[str, Any], + deadline: float, +) -> tuple[list[ReconcileAction], bool]: + """ + Reconciles every pending write. Returns ``(actions, timed out mid-run)``. + + The loop never aborts: a per-skill failure becomes an ``error`` action and the + next skill is attempted, because returning early would skip the caller's + manifest rewrite and orphan every file already written in this run. + """ + actions: list[ReconcileAction] = [] + timed_out = False + + for request in requests: + if request.skill is None: + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=request.error + or f"skill '{request.key}' could not be resolved", + ) + ) + continue + if time.monotonic() >= deadline: + timed_out = True + actions.append( + ReconcileAction( + key=request.key, + action="error", + error=( + "the timeout was exhausted before skill " + f"'{request.key}' could be written" + ), + ) + ) + continue + try: + actions.append(_write_one(root, request.skill, entries)) + except OSError as exc: + # A safety net, not the primary defense. pathlib's stat probes swallow + # only ENOENT/ENOTDIR/EBADF/ELOOP and re-raise every other errno, so an + # unexpected filesystem condition must not abort the loop. + actions.append( + ReconcileAction( + key=request.skill.key, + action="error", + version=request.skill.version, + error=f"skill '{request.skill.key}' could not be reconciled: {exc}", + ) + ) + + return actions, timed_out + + +def _rewrite_manifest( + root: Path, manifest: dict[str, Any], entries: dict[str, Any] +) -> list[ReconcileAction]: + """Writes the updated manifest. Returns an error action, or nothing.""" + manifest["manifestVersion"] = MANIFEST_VERSION + manifest["entries"] = entries + try: + # json.dumps is inside the guard: indent= selects the pure-Python encoder, + # and unknown fields must be round-tripped, so a deeply nested + # planted field can raise RecursionError here — after every skill file is + # already on disk. + serialized = json.dumps(manifest, indent=2, sort_keys=True).encode("utf-8") + atomic_write_in(root, MANIFEST_FILENAME, serialized) + except Exception as exc: + return [_run_error(f"the skills manifest could not be written: {exc}")] + return [] + + +# ------------------------------------------------------------------------- +# Request resolution — content in, or a reason there is none +# ------------------------------------------------------------------------- + + +def _unavailable(reason: str) -> str: + """Wraps *reason* as a retrieval-unavailable message.""" + return f"{_UNAVAILABLE_PREFIX}{reason}" + + +@dataclass(frozen=True) +class _RetrievalBlocked: + """Why retrieval must not be attempted. The reason is caller-facing.""" + + reason: str + + +def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBlocked: + """ + The configured store, or why retrieval must not be attempted. + + Written once because this gate is what sets ``unavailable`` and therefore + suppresses pruning. If it were maintained in two places, a condition added + to one and not the other would not merely produce a wrong message — it + would delete the user's files. + """ + if time.monotonic() >= deadline: + return _RetrievalBlocked( + _unavailable( + f"the timeout was exhausted before {subject} could be retrieved" + ) + ) + store = get_store() + if store is None: + return _RetrievalBlocked(_unavailable(NO_STORE_MESSAGE)) + return store + + +def _resolve_requests( + skills: Sequence[Skill | SkillReference | str] | str, + deadline: float, + on_unavailable: OnUnavailable, +) -> tuple[list[_PendingWrite], bool]: + """ + Turns the caller's input into one request per skill. + + Returns the requests plus whether any retrieval was left incomplete — an + absent store, a raising store, or an exhausted timeout. That flag suppresses + pruning: deleting managed files because retrieval failed would turn a + transport outage into data loss. + """ + if isinstance(skills, str): + if skills != "*": + raise ValueError( + 'write_skills takes a sequence of skills or the literal "*"; ' + f"got {skills!r}" + ) + return _resolve_all(deadline, on_unavailable) + + requests: list[_PendingWrite] = [] + incomplete = False + for item in skills: + if isinstance(item, Skill): + requests.append(_PendingWrite(key=item.key, skill=item)) + continue + + key, wanted = reference_target(item) + resolved = _resolve_reference(key, wanted, deadline) + if resolved.unavailable: + incomplete = True + if on_unavailable == "raise": + raise RuntimeError(resolved.error) + requests.append( + _PendingWrite(key=key, skill=resolved.skill, error=resolved.error) + ) + + return requests, incomplete + + +def _resolve_reference( + key: str, wanted_version: int | None, deadline: float +) -> Resolution: + """ + Resolves one reference for the materialization path. + + Same core as the accessors, plus the two conditions only this path treats as + data rather than as an exception: an exhausted deadline and an absent store. + """ + store = _available_store(deadline, f"'{key}'") + if isinstance(store, _RetrievalBlocked): + return Resolution(error=store.reason, unavailable=True) + + resolved = resolve_from_store(store, key, wanted_version) + if resolved.unavailable and resolved.error is not None: + return Resolution(error=_unavailable(resolved.error), unavailable=True) + return resolved + + +def _unavailable_run( + error: str, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """ + One run-level retrieval failure — raised, or reported against the empty key. + + Always reports the run incomplete, which is what suppresses pruning: nothing + was retrieved, so every managed file on disk has to be assumed current. + """ + if on_unavailable == "raise": + raise RuntimeError(error) + return [_PendingWrite(key="", error=error)], True + + +def _pending_for_raw(object_key: str, raw: Any) -> _PendingWrite: + """ + One raw store object as a pending write — verified, or reported as failed. + + Present but unverifiable is NOT the same as revoked. Dropping it silently + would leave the key out of the requested set, so prune would delete the last + known-good copy already on disk and report a routine "removed" with + report.ok still true. A failed request instead gets the same treatment the + reference path already gives (see ``_resolve_reference``): the outcome is + surfaced, and the key stays in the requested set so nothing is pruned. + """ + skill = verify_raw_skill(raw) + if skill is not None: + return _PendingWrite(key=skill.key, skill=skill) + # The on-disk copy lives under the object's *own* key, which a custom store + # may key differently in ``all_objects``. The failure must be recorded under + # the object's key, or the copy written under it on an earlier run would + # fall out of the requested set and be pruned — the very deletion this + # function exists to prevent. + raw_key = raw.get("key") if isinstance(raw, dict) else None + key = raw_key if is_valid_skill_key(raw_key) else object_key + if not is_valid_skill_key(key): + # Neither key is usable, so this failure cannot be attributed to a skill + # — the run-level sentinel is the honest report. + return _PendingWrite( + key=_RUN_LEVEL_KEY, + error="the skill store served an object under an invalid key; " + "it was withheld", + ) + return _PendingWrite( + key=key, + error=f"skill '{key}' failed integrity verification and was " + "withheld; the copy already on disk was left alone", + ) + + +def _resolve_all( + deadline: float, on_unavailable: OnUnavailable +) -> tuple[list[_PendingWrite], bool]: + """Resolves the ``"*"`` form — everything the store currently holds.""" + store = _available_store(deadline, "the skill set") + if isinstance(store, _RetrievalBlocked): + return _unavailable_run(store.reason, on_unavailable) + + # Deliberately not via all_skills(), which reports a raising store as an + # empty result — that would look like "every skill was revoked" and let + # prune delete the lot. + objects, error = list_raw_objects(store) + if error is not None: + return _unavailable_run(_unavailable(error), on_unavailable) + + # One object per key, at its newest version. ``all_objects`` may hold several + # versions of one key, and //SKILL.md is a single path — writing it + # twice in one run is a bug rather than a policy. + candidates = newest_by_key(objects) + requests = [_pending_for_raw(key, raw) for key, raw in candidates] + log_withholding_summary( + "skills held by the store", + len(requests), + sum(1 for request in requests if request.skill is not None), + ) + return requests, False + + +# ------------------------------------------------------------------------- +# The managed root and its manifest +# ------------------------------------------------------------------------- + + +def _resolve_root(root: str | os.PathLike[str]) -> Path: + """ + Resolves the managed root once, up front. + + An unusable root is a caller error rather than a per-skill outcome, so this + raises. Only the leaf directory is ever created — recursively creating + missing ancestors would let a typo scatter a directory tree. + """ + path = Path(os.fspath(root)) + + # pathlib re-raises any errno outside ENOENT/ENOTDIR/EBADF/ELOOP, so an + # unreadable parent would surface as PermissionError where the docs + # promise ValueError. + try: + is_symlink = path.is_symlink() + exists = path.exists() + is_dir = path.is_dir() + except OSError as exc: + raise ValueError(f"the skills root could not be inspected: {exc}") from exc + + if is_symlink: + raise ValueError( + f"the skills root must be a real directory, not a symlink: {path}" + ) + + if exists: + if not is_dir: + raise ValueError(f"the skills root is not a directory: {path}") + else: + parent = path.parent + try: + parent_is_dir = parent.is_dir() + except OSError as exc: + raise ValueError( + f"the parent of the skills root could not be inspected: {exc}" + ) from exc + if not parent_is_dir: + raise ValueError( + f"the parent of the skills root does not exist: {parent}. " + "write_skills creates only the leaf directory." + ) + try: + path.mkdir() + except OSError as exc: + raise ValueError(f"the skills root could not be created: {exc}") from exc + + return Path(os.path.realpath(path)) + + +def _load_manifest(root: Path) -> tuple[dict[str, Any], str | None]: + """ + Loads the manifest. Returns ``(manifest, error)``. + + A manifest that cannot be read, cannot be parsed, is not an object, carries a + ``manifestVersion`` this release does not understand, or has a malformed + ``entries`` map is **corrupt**. The caller then performs no destructive + action and leaves the file itself alone: rewriting it would destroy the only + record of what the SDK owns, and acting on a manifest we cannot read would + mean guessing at which of the customer's files are ours. + + An absent manifest is not corrupt — that is simply a fresh root. + """ + path = root / MANIFEST_FILENAME + fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} + + if not path.exists(): + return fresh, None + + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # UnicodeDecodeError is a ValueError, not an OSError: non-UTF-8 bytes in + # the manifest are corruption, and must fail closed like any other. + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + + try: + data = json.loads(text) + except (ValueError, RecursionError) as exc: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not valid JSON ({exc}); " + "refusing every destructive action" + ) + + if not isinstance(data, dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is not a JSON object; " + "refusing every destructive action" + ) + + version = data.get("manifestVersion") + if ( + not isinstance(version, int) + or isinstance(version, bool) + or version > MANIFEST_VERSION + ): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} declares manifestVersion " + f"{version!r}, which this SDK cannot read; refusing every destructive " + "action" + ) + + if not isinstance(data.get("entries"), dict): + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} has a malformed 'entries' " + "map; refusing every destructive action" + ) + + return data, None + + +# ------------------------------------------------------------------------- +# Per-skill reconcile +# ------------------------------------------------------------------------- + + +def _unsafe_path_reason( + root: Path, skill_dir: Path, target: Path, key: str, *, require_directory: bool +) -> str | None: + """ + The path defenses, in one place. + + Returns why ``//SKILL.md`` must not be touched, or ``None``. + Shared by the write and prune paths: ``agents.md`` marks these checks + non-relaxable, and maintaining them twice is how they drift. + + *require_directory* is the one genuine difference between the two callers. A + write needs a real directory to write into. A prune only needs to not follow + a link — an entry whose directory has been replaced by a plain file has + already lost the file this SDK owned, so reporting ``removed`` is what lets + the stale manifest entry be dropped rather than pinned forever. + + Note that the containment check is unconditional even though ``skill_dir`` + may not exist yet: ``realpath`` resolves the existing prefix and appends the + rest, so a fresh key under a valid root passes. + """ + if skill_dir.is_symlink(): + return f"{key} is a symlink" + if require_directory and skill_dir.exists() and not skill_dir.is_dir(): + return f"{key} exists and is not a directory" + if target.is_symlink(): + return "the target file is a symlink" + if Path(os.path.realpath(skill_dir)).parent != root: + return f"it resolves outside the managed root {root}" + return None + + +def _key_rejection_reason(key: Any) -> str | None: + """ + Why *key* must not become a directory name under the managed root, or ``None``. + + Re-validated locally whatever any upstream layer already did, and + before any filesystem call, because a key becomes a path component. Shared by + the write and the prune paths so the two cannot disagree about which keys + this SDK could own; ``agents.md`` marks these checks non-relaxable, and + maintaining them twice is how they drift. + + ``key.encode`` is safe here only because it runs *after* the pattern check: + the key grammar admits no surrogate, so there is no unencodable key left to + raise on. Do not reorder these two. + """ + if not is_valid_skill_key(key): + return f"{key!r} is not a valid skill key: it {skill_key_rejection_reason(key)}" + # The data model allows 256 characters; no mainstream filesystem allows a + # 256-byte path component. Catch it here so it is a reported action rather + # than an ENAMETOOLONG raised from the first stat in the caller. + key_bytes = len(key.encode("utf-8")) + if key_bytes > _MAX_PATH_COMPONENT_BYTES: + return ( + f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " + f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" + ) + return None + + +def _write_one(root: Path, skill: Skill, entries: dict[str, Any]) -> ReconcileAction: + """Reconciles one verified skill against the managed root.""" + key = skill.key + + def failed(message: str) -> ReconcileAction: + return ReconcileAction( + key=key, action="error", version=skill.version, error=message + ) + + rejection = _key_rejection_reason(key) + if rejection is not None: + return failed(f"{rejection}; nothing was written") + if not is_valid_skill_version(skill.version): + return failed( + f"skill '{key}' has version {skill.version!r}, which is not an " + "integer >= 1; nothing was written" + ) + + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + relative = f"{key}/{SKILL_FILENAME}" + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=True) + if unsafe is not None: + return failed(f"'{relative}' was refused: {unsafe}; nothing was written") + + # Re-verify immediately before writing, through the same core the accessors + # use: a Skill can also be constructed directly by a caller. + verified = verified_bytes(key, skill.content, skill.content_hash, skill.version) + if isinstance(verified, VerificationFailure): + return failed( + f"skill '{key}' failed verification immediately before writing: " + f"{verified.reason}; nothing was written" + ) + encoded, content_hash = verified.encoded, verified.content_hash + + # Overwrite only what the manifest records as ours under this key. + entry = entries.get(relative) + managed = isinstance(entry, dict) and entry.get("key") == key + exists = target.exists() + + if exists and not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) + + if exists: + try: + on_disk = _read_regular_file(target) + except OSError as exc: + return failed(f"'{relative}' could not be read: {exc}") + + if hashlib.sha256(on_disk).hexdigest() == content_hash: + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, "skipped_current") + return ReconcileAction( + key=key, + action="skipped_current", + version=skill.version, + path=str(target), + ) + # Stale version or local tampering — LD-resolved content wins. + action: ReconcileActionKind = "updated" + else: + action = "written" + + write_error = _write_through_descriptor(skill_dir, encoded, key, relative) + if write_error is not None: + return failed(write_error) + + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, action) + return ReconcileAction( + key=key, action=action, version=skill.version, path=str(target) + ) + + +def _read_regular_file(target: Path) -> bytes: + """ + Reads *target*, refusing anything that is not a regular file. + + A plain ``Path.read_bytes`` would ``open()`` by name — and opening a FIFO + with no writer blocks forever, so an attacker who can swap the managed file + for one (the same capability the symlink checks defend against) could hang + the whole reconcile, and the event loop with it. ``O_NONBLOCK`` makes that + open return immediately (it is a no-op for regular files), ``O_NOFOLLOW`` + refuses a trailing symlink, and the ``fstat`` on the descriptor — not the + path — is what the type check trusts. ``O_BINARY`` is what keeps these + bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor + without it translates CRLF on read, which would fail the hash comparison + against content that is actually current. + """ + flags = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_NONBLOCK", 0) + | getattr(os, "O_BINARY", 0) + ) + fd = os.open(target, flags) + try: + if not stat.S_ISREG(os.fstat(fd).st_mode): + raise OSError("the target file is not a regular file") + chunks: list[bytes] = [] + while True: + chunk = os.read(fd, 65536) + if not chunk: + return b"".join(chunks) + chunks.append(chunk) + finally: + os.close(fd) + + +def _write_through_descriptor( + skill_dir: Path, encoded: bytes, key: str, relative: str +) -> str | None: + """ + Performs the write itself. Returns a failure reason, or ``None`` on success. + + Split out of ``_write_one`` because everything above it decides *whether* to + write and this decides nothing: the directory is pinned to a descriptor and + every remaining step is relative to it, so none of the checks above can be + invalidated by a swap between here and the rename. + """ + try: + with pinned_directory(skill_dir, create=True) as dir_fd: + try: + atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=dir_fd) + except OSError as exc: + return f"'{relative}' could not be written: {exc}" + except OSError as exc: + return f"the directory for skill '{key}' could not be created: {exc}" + except ValueError as exc: + return f"'{relative}' was refused: {exc}" + return None + + +def _update_entry( + entries: dict[str, Any], relative: str, skill: Skill, content_hash: str +) -> None: + """ + Records a managed path in the manifest. + + Merges into any existing entry rather than replacing it, so fields written by + a future SDK release survive this one's rewrite. + + ``sha256`` and ``writtenAt`` are recorded for forensics only: the reconcile + decides currency by hashing the bytes on disk, precisely because the + manifest is untrusted, so neither field is ever read back as a decision + input. + """ + existing = entries.get(relative) + entry = dict(existing) if isinstance(existing, dict) else {} + entry["key"] = skill.key + entry["version"] = skill.version + entry["sha256"] = content_hash + entry["writtenAt"] = _utc_timestamp() + entries[relative] = entry + + +# ------------------------------------------------------------------------- +# Pruning — how revocation takes effect +# ------------------------------------------------------------------------- + + +def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction: + """ + A prune refusal. Mirrors ``_write_one``'s local ``failed`` helper. + + *version* comes off the manifest, which is untrusted, so it is validated here + rather than at each call site — the same guard the ``removed`` action applies, + so a refusal and a removal report the field identically. + Callers that genuinely do not know a version pass nothing; none of them may + invent one. + """ + return ReconcileAction( + key=key, + action="error", + version=version if is_valid_skill_version(version) else None, + error=message, + ) + + +def _prune( + root: Path, entries: dict[str, Any], requested: set[str], deadline: float +) -> list[ReconcileAction]: + """ + Removes managed skills that are no longer requested. + + This is also how revocation takes effect: a revoked skill is simply absent + from the resolved set, so the next reconcile removes it. There is + deliberately no opt-out. + + The deadline applies here just as it does to the writes: a skill left + unpruned is reported as an error and stays in the manifest, so the next + reconcile picks it up. + """ + actions: list[ReconcileAction] = [] + + for relative, entry in list(entries.items()): + if not isinstance(entry, dict): + continue + key = entry.get("key") + if not isinstance(key, str) or key in requested: + continue + + if time.monotonic() >= deadline: + actions.append( + _prune_error( + key, + f"the timeout was exhausted before '{relative}' could be " + "pruned; it was left in place", + entry.get("version"), + ) + ) + continue + + # Only a manifest path this SDK could have written is removable. + if ( + _key_rejection_reason(key) is not None + or relative != f"{key}/{SKILL_FILENAME}" + ): + actions.append( + _prune_error( + key, + f"manifest entry '{relative}' does not name a path this SDK " + f"could own under key '{key}'; it was left in place", + entry.get("version"), + ) + ) + continue + + try: + actions.append(_prune_one(root, relative, key, entries)) + except OSError as exc: + actions.append( + _prune_error( + key, + f"'{relative}' could not be removed: {exc}", + entry.get("version"), + ) + ) + + return actions + + +def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: + """ + Performs the removal itself. Returns a failure reason, or ``None`` on success. + + The mirror of ``_write_through_descriptor``, and split out for the same + reason: everything above it decides *whether* to remove, and this decides + nothing. The directory is pinned before the unlink because unlink never + follows a trailing symlink but does resolve the directory above it, so a + ``/`` swapped for a symlink between the checks and here would + otherwise delete a file outside the root. + """ + try: + with pinned_directory(skill_dir) as dir_fd: + try: + unlink_file(skill_dir, SKILL_FILENAME, dir_fd=dir_fd) + except SymlinkRefused: + return f"'{relative}' was not removed: the target file is a symlink" + except OSError as exc: + return f"'{relative}' could not be removed: {exc}" + except ValueError as exc: + return f"'{relative}' was not removed: {exc}" + return None + + +def _prune_one( + root: Path, relative: str, key: str, entries: dict[str, Any] +) -> ReconcileAction: + """Removes one managed skill file, and its directory when that empties it.""" + skill_dir = root / key + target = skill_dir / SKILL_FILENAME + version = entries[relative].get("version") + + unsafe = _unsafe_path_reason(root, skill_dir, target, key, require_directory=False) + if unsafe is not None: + return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + + removed_from_disk = False + if target.exists(): + failure = _unlink_through_descriptor(skill_dir, relative) + if failure is not None: + return _prune_error(key, failure, version) + removed_from_disk = True + try: + # Path-based, and safe that way: rmdir never follows a trailing + # symlink (it fails ENOTDIR) and only ever succeeds on an empty + # directory. + skill_dir.rmdir() + except OSError: + pass # the customer keeps their own files here too + + entries.pop(relative, None) + + if removed_from_disk: + record_revoked(key, version) + + return ReconcileAction( + key=key, + action="removed", + version=version if is_valid_skill_version(version) else None, + path=str(target), + ) + + +def _utc_timestamp() -> str: + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 4249d5cb..10c90d8c 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,54 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +ReconcileActionKind = Literal[ + "written", "updated", "skipped_current", "removed", "error" +] +"""The closed set of outcomes ``write_skills`` reports.""" + + +@dataclass(frozen=True) +class ReconcileAction: + """What ``write_skills`` did — or refused to do — for one skill.""" + + key: str + """ + The skill key, or the **empty string** for a failure that belongs to the run + rather than to one skill — a corrupt manifest, a manifest that could not be + rewritten, a retrieval that failed before any key was known. Callers grouping + a report by key need to expect that sentinel; a report may carry both kinds. + """ + action: ReconcileActionKind + version: int | None = None + path: str | None = None + """Canonical resolved path, when one was determined.""" + error: str | None = None + """Failure detail, set only when ``action == "error"``.""" + + +@dataclass(frozen=True) +class ReconcileReport: + """The result of a ``write_skills`` run — every outcome is visible here.""" + + actions: list[ReconcileAction] = field(default_factory=list) + + @property + def ok(self) -> bool: + """``True`` iff no action is an ``error``.""" + return not self.errors + + @property + def errors(self) -> list[ReconcileAction]: + """ + The ``error`` actions, in ``actions`` order. + + Exposed so callers never re-derive it — filtering ``actions`` is + boilerplate that otherwise reappears in every consumer. ``ok`` is defined + in terms of this, so the two can never disagree. + """ + return [a for a in self.actions if a.action == "error"] + + # --------------------------------------------------------------------------- # Model / graph options # --------------------------------------------------------------------------- diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 203eeb4f..0c656ede 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -17,6 +17,8 @@ import launchdarkly_ai_server.skills as skills_module from launchdarkly_ai_server import ( InMemorySkillStore, + ReconcileAction, + ReconcileReport, Skill, SkillReference, all_skills, @@ -129,7 +131,7 @@ def _fabricated_hash_cases() -> list[Any]: class TestSkillTypes: - """Immutability and optional metadata.""" + """Immutability, optional metadata, and ``ReconcileReport.ok``.""" def test_skill_reference_is_immutable(self) -> None: ref = SkillReference(key="pdf-extraction", version=2) @@ -164,6 +166,70 @@ def test_skill_metadata_defaults_to_none(self) -> None: assert skill.name is None assert skill.description is None + def test_report_ok_true_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="skipped_current", version=2), + ReconcileAction(key="c", action="removed"), + ReconcileAction(key="d", action="updated", version=3), + ] + ) + assert report.ok is True + + def test_report_ok_false_with_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + assert report.ok is False + + def test_empty_report_is_ok(self) -> None: + assert ReconcileReport(actions=[]).ok is True + + def test_report_errors_lists_error_actions_in_order(self) -> None: + """The report exposes its error actions itself.""" + first = ReconcileAction(key="b", action="error", error="first") + second = ReconcileAction(key="d", action="error", error="second") + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + first, + ReconcileAction(key="c", action="skipped_current", version=2), + second, + ReconcileAction(key="e", action="removed"), + ] + ) + assert report.errors == [first, second] + + def test_report_errors_empty_when_no_error_action(self) -> None: + report = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="removed"), + ] + ) + assert report.errors == [] + + def test_empty_report_has_no_errors(self) -> None: + assert ReconcileReport(actions=[]).errors == [] + + def test_report_ok_and_errors_always_agree(self) -> None: + """``ok`` is true iff ``errors`` is empty, on the same objects.""" + clean = ReconcileReport( + actions=[ReconcileAction(key="a", action="written", version=1)] + ) + failed = ReconcileReport( + actions=[ + ReconcileAction(key="a", action="written", version=1), + ReconcileAction(key="b", action="error", error="nope"), + ] + ) + for report in (clean, failed, ReconcileReport(actions=[])): + assert report.ok is (report.errors == []) + class TestSkillRefs: """Pure projection of the config's skills array.""" @@ -267,6 +333,59 @@ def test_object_kind_is_not_public_api(self) -> None: assert "SKILL_OBJECT_KIND" not in package.__all__ assert not hasattr(package, "SKILL_OBJECT_KIND") + def test_constants_are_exported_from_the_package_root(self) -> None: + import launchdarkly_ai_server as package + + assert package.SKILL_FILENAME == "SKILL.md" + assert package.MANIFEST_FILENAME == ".launchdarkly-skills.json" + assert package.MANIFEST_VERSION == 1 + + def test_constants_are_listed_in_dunder_all(self) -> None: + """A name absent from ``__all__`` is not part of the public surface.""" + import launchdarkly_ai_server as package + + expected = { + "SKILL_FILENAME", + "MANIFEST_FILENAME", + "MANIFEST_VERSION", + } + assert expected <= set(package.__all__) + + def test_closed_set_types_are_exported_from_the_package_root(self) -> None: + """The two closed-set unions are public API, not implementation detail. + + ``ReconcileActionKind`` types the ``ReconcileAction.action`` field every + consumer of a report reads and switches on, and ``OnUnavailable`` types + a public keyword argument of ``write_skills``. ``agents.md`` forbids + handler packages from importing sub-path modules, so a name exported + only from the implementation module has no supported import path. + """ + import launchdarkly_ai_server as package + + assert hasattr(package, "ReconcileActionKind") + assert hasattr(package, "OnUnavailable") + assert {"ReconcileActionKind", "OnUnavailable"} <= set(package.__all__) + + def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: + """The union must match the actions a report can actually carry. + + Spelled out rather than imported from the implementation for the same + reason as the constants above: deriving the expectation from the thing + under test would make the assertion circular. + """ + import typing + + import launchdarkly_ai_server as package + + assert set(typing.get_args(package.ReconcileActionKind)) == { + "written", + "updated", + "skipped_current", + "removed", + "error", + } + assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" import launchdarkly_ai_server as package @@ -811,7 +930,7 @@ class TestWithholdingSummary: """ def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: - raw = make_raw_skill(key=key) + raw: dict[str, Any] = make_raw_skill(key=key) raw["contentHash"] = "0" * 64 return raw diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py new file mode 100644 index 00000000..62d01d59 --- /dev/null +++ b/packages/client/tests/test_skills_fs.py @@ -0,0 +1,582 @@ +""" +Tests for ``write_skills`` — filesystem materialization and manifest reconcile +semantics. + +Every test writes only inside pytest's ``tmp_path``. No network, no real +LaunchDarkly client, no real skill transport. + +The security abuse matrix — path traversal, symlink attacks, clobber +protection, corrupt manifests, atomicity under an injected crash, and the +materialization telemetry allowlist — is a separate module. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +import launchdarkly_ai_server.skills as skills_module +from launchdarkly_ai_server import ( + InMemorySkillStore, + Skill, + SkillReference, + write_skills, +) + +MANIFEST_NAME = ".launchdarkly-skills.json" +SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" + +INTEGRITY_SIGNAL = "AgentControl Skill Integrity Failure" + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + r = tmp_path / "skills" + r.mkdir() + return r + + +pytestmark = pytest.mark.usefixtures("reset_skill_state") +"""Every test in this module runs against freshly cleared module state.""" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _skill( + key: str = "test-skill", version: int = 1, content: str = SKILL_BODY +) -> Skill: + return Skill( + key=key, + version=version, + content=content.encode("utf-8"), + content_hash=_hash(content), + ) + + +def _manifest_path(root: Path) -> Path: + return root / MANIFEST_NAME + + +def _read_manifest(root: Path) -> dict[str, Any]: + return json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + +def _write_manifest(root: Path, raw: Any) -> None: + root.mkdir(parents=True, exist_ok=True) + _manifest_path(root).write_text( + raw if isinstance(raw, str) else json.dumps(raw), encoding="utf-8" + ) + + +def _entry(key: str, version: int, content: str) -> dict[str, Any]: + return { + "key": key, + "version": version, + "sha256": _hash(content), + "writtenAt": "2026-08-14T19:00:00Z", + } + + +def _place_managed(root: Path, key: str, content: str, version: int = 1) -> Path: + """Pre-create a file AND its manifest entry — i.e. an SDK-managed path.""" + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {f"{key}/SKILL.md": _entry(key, version, content)}, + }, + ) + return target + + +def _actions_by_key(report: Any) -> dict[str, Any]: + return {a.key: a for a in report.actions} + + +def _error_messages(report: Any) -> list[str]: + """All ``error`` action messages, regardless of which key they hang off. + + Run-level (manifest) errors have no well-defined ``key`` yet, so assertions + about them scan every error action rather than looking one up by key. + """ + return [a.error or "" for a in report.actions if a.action == "error"] + + +class TestBasicWrites: + """Basic writes and the returned report.""" + + async def test_new_skill_is_written_verbatim(self, root: Path) -> None: + report = await write_skills([_skill("pdf-extraction", 2)], root) + + target = root / "pdf-extraction" / "SKILL.md" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert report.ok is True + action = _actions_by_key(report)["pdf-extraction"] + assert action.action == "written" + assert action.version == 2 + assert action.path is not None + assert Path(action.path).resolve() == target.resolve() + assert action.error is None + + async def test_skill_inputs_need_no_store(self, root: Path) -> None: + report = await write_skills([_skill("a")], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_reference_inputs_resolve_through_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=3)) + report = await write_skills([SkillReference(key="a", version=3)], root) + assert report.ok is True + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_string_inputs_resolve_latest( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a", version=9)) + report = await write_skills(["a"], root) + assert _actions_by_key(report)["a"].version == 9 + + async def test_star_writes_everything_in_the_store( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + for k in ("a", "b", "c"): + store.put(make_raw_skill(key=k)) + report = await write_skills("*", root) + assert report.ok is True + assert len([a for a in report.actions if a.action == "written"]) == 3 + for k in ("a", "b", "c"): + assert (root / k / "SKILL.md").exists() + + async def test_one_action_per_requested_skill(self, root: Path) -> None: + report = await write_skills([_skill("a"), _skill("b")], root) + assert sorted(a.key for a in report.actions) == ["a", "b"] + + async def test_empty_request_on_empty_root_is_ok(self, root: Path) -> None: + report = await write_skills([], root) + assert report.ok is True + assert report.actions == [] + + +class TestManifest: + """Manifest format and forward compatibility.""" + + async def test_manifest_format_is_exact(self, root: Path) -> None: + await write_skills([_skill("pdf-extraction", 2)], root) + + manifest = _read_manifest(root) + assert manifest["manifestVersion"] == 1 + entry = manifest["entries"]["pdf-extraction/SKILL.md"] + assert entry["key"] == "pdf-extraction" + assert entry["version"] == 2 + assert entry["sha256"] == _hash(SKILL_BODY) + assert isinstance(entry["writtenAt"], str) + + async def test_entry_paths_are_forward_slash_relative(self, root: Path) -> None: + await write_skills([_skill("a")], root) + keys = list(_read_manifest(root)["entries"].keys()) + assert keys == ["a/SKILL.md"] + assert "\\" not in keys[0] + assert not keys[0].startswith("/") + + async def test_unknown_fields_are_preserved_on_rewrite(self, root: Path) -> None: + entry = _entry("a", 1, SKILL_BODY) + entry["futureEntryField"] = "keep-me" + _write_manifest( + root, + { + "manifestVersion": 1, + "futureTopLevelField": {"keep": True}, + "entries": {"a/SKILL.md": entry}, + }, + ) + (root / "a").mkdir() + (root / "a" / "SKILL.md").write_text(SKILL_BODY, encoding="utf-8") + + await write_skills([_skill("a", 2, SKILL_BODY + "more\n")], root) + + manifest = _read_manifest(root) + assert manifest["futureTopLevelField"] == {"keep": True} + assert manifest["entries"]["a/SKILL.md"]["futureEntryField"] == "keep-me" + + +class TestReconcileSemantics: + """The reconcile state table.""" + + async def test_unchanged_managed_file_is_skipped_current(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + before = target.stat().st_mtime_ns + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "skipped_current" + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert target.stat().st_mtime_ns == before + + async def test_new_version_updates(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY, version=1) + new_content = SKILL_BODY + "second version\n" + + report = await write_skills([_skill("a", 2, new_content)], root) + + action = _actions_by_key(report)["a"] + assert action.action == "updated" + assert action.version == 2 + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == new_content + assert _read_manifest(root)["entries"]["a/SKILL.md"]["version"] == 2 + + async def test_local_tampering_is_overwritten(self, root: Path) -> None: + target = _place_managed(root, "a", SKILL_BODY) + target.write_text("locally tampered\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert _actions_by_key(report)["a"].action == "updated" + assert target.read_text(encoding="utf-8") == SKILL_BODY + + async def test_prune_removes_formerly_managed_skill(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root) + + assert _actions_by_key(report)["gone"].action == "removed" + assert not (root / "gone" / "SKILL.md").exists() + assert not (root / "gone").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_prune_false_keeps_the_file(self, root: Path) -> None: + target = _place_managed(root, "gone", SKILL_BODY) + + report = await write_skills([], root, prune=False) + + assert target.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + assert "gone/SKILL.md" in _read_manifest(root)["entries"] + + async def test_prune_does_not_touch_unmanaged_files(self, root: Path) -> None: + _place_managed(root, "gone", SKILL_BODY) + bystander = root / "user-notes.md" + bystander.write_text("mine\n", encoding="utf-8") + user_dir_file = root / "user-skill" / "SKILL.md" + user_dir_file.parent.mkdir() + user_dir_file.write_text("hand written\n", encoding="utf-8") + + await write_skills([], root) + + assert bystander.read_text(encoding="utf-8") == "mine\n" + assert user_dir_file.read_text(encoding="utf-8") == "hand written\n" + + async def test_prune_refusal_for_unownable_path_reports_the_version( + self, root: Path + ) -> None: + """A prune refusal carries the manifest's version. + + A manifest entry whose path is not one this SDK could have written is + refused rather than removed. The entry is in hand at that point, so the + error action must carry its version — otherwise a prune *failure* is + strictly less informative than a prune *success*, which does report it. + """ + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": { + # Right key, wrong filename — not a path this SDK could own. + "orphan/NOTES.md": _entry("orphan", 7, SKILL_BODY), + }, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["orphan"] + assert action.action == "error" + assert action.version == 7 + + async def test_prune_refusal_for_symlinked_target_reports_the_version( + self, root: Path + ) -> None: + """Same contract on the symlink refusal path (prune side).""" + if not hasattr(os, "symlink"): + pytest.skip("platform has no symlink support") + (root / "a").mkdir() + outside_file = root.parent / "victim.md" + outside_file.write_text("victim content\n", encoding="utf-8") + (root / "a" / "SKILL.md").symlink_to(outside_file) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {"a/SKILL.md": _entry("a", 4, "victim content\n")}, + }, + ) + + report = await write_skills([], root) + + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.version == 4 + + async def test_unresolvable_request_still_reports_no_version( + self, root: Path + ) -> None: + """The other half of the contract: do not invent a version. + + A reference that could not be retrieved has neither a manifest entry + nor a ``Skill``, so there is no version to report and ``version`` stays + ``None``. Without this, "always populate version" would be satisfied by + fabricating one. + """ + report = await write_skills([SkillReference(key="ghost", version=3)], root) + + action = _actions_by_key(report)["ghost"] + assert action.action == "error" + assert action.version is None + + async def test_prune_keeps_directory_when_not_empty(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + extra = root / "a" / "user-file.txt" + extra.write_text("keep\n", encoding="utf-8") + + report = await write_skills([], root) + + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a" / "SKILL.md").exists() + assert extra.exists() + + +class TestRootHandling: + """Root resolution.""" + + async def test_absent_leaf_root_is_created(self, tmp_path: Path) -> None: + target_root = tmp_path / "skills" + report = await write_skills([_skill("a")], target_root) + assert report.ok is True + assert (target_root / "a" / "SKILL.md").exists() + + async def test_missing_ancestors_raise(self, tmp_path: Path) -> None: + with pytest.raises(ValueError): + await write_skills([_skill("a")], tmp_path / "a" / "b" / "c") + + async def test_root_that_is_a_file_raises(self, tmp_path: Path) -> None: + file_root = tmp_path / "not-a-dir" + file_root.write_text("x", encoding="utf-8") + with pytest.raises(ValueError): + await write_skills([_skill("a")], file_root) + + async def test_accepts_string_root(self, root: Path) -> None: + report = await write_skills([_skill("a")], str(root)) + assert report.ok is True + + +class TestSkillsArgumentErrors: + """A bare string that is not ``"*"`` raises. + + A ``ValueError``, not a ``TypeError``: a string *is* an accepted argument + type here, since ``"*"`` means "everything the store holds", so this is an + acceptable type carrying an invalid value. The accessors' equivalent guard + is a ``TypeError`` because a string is never a valid argument there. + """ + + async def test_bare_non_star_string_raises_value_error(self, root: Path) -> None: + with pytest.raises(ValueError) as excinfo: + await write_skills("pdf-extraction", root) + + # Naming the accepted forms is the actionable half of the message. + assert '"*"' in str(excinfo.value) + + async def test_star_is_accepted(self, root: Path) -> None: + """Positive control — otherwise the guard above could reject every string.""" + store = InMemorySkillStore() + store.put( + { + "key": "a", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + ) + skills_module._set_store(store) + + report = await write_skills("*", root) + + assert report.ok is True + assert (root / "a" / "SKILL.md").exists() + + async def test_bare_string_writes_nothing(self, root: Path) -> None: + """The raise precedes any filesystem work. + + Asserting only the raise would also pass for an implementation that + created one directory per character before failing. + """ + with pytest.raises(ValueError): + await write_skills("abc", root) + + assert list(root.iterdir()) == [] + + +class TestResilience: + """Unavailable retrieval and timeout.""" + + async def test_keep_is_the_default_and_does_not_raise(self, root: Path) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert existing.read_text(encoding="utf-8") == SKILL_BODY + + async def test_raise_mode_propagates(self, root: Path) -> None: + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|skill store)"): + await write_skills( + [SkillReference(key="a", version=1)], root, on_unavailable="raise" + ) + + async def test_store_error_is_reported_not_raised( + self, root: Path, exploding_store: Any + ) -> None: + report = await write_skills([SkillReference(key="a", version=1)], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + + async def test_exhausted_timeout_behaves_as_unavailable( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + report = await write_skills( + [SkillReference(key="a", version=1)], root, timeout=0 + ) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_exhausted_timeout_raises_in_raise_mode( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="a")) + with pytest.raises(RuntimeError, match=r"(?i)(unavailable|timeout|timed out)"): + await write_skills( + [SkillReference(key="a", version=1)], + root, + timeout=0, + on_unavailable="raise", + ) + + async def test_exhausted_timeout_stops_pruning( + self, root: Path, store: InMemorySkillStore + ) -> None: + """The deadline bounds pruning too, not just retrieval and the writes. + + A run whose writes all land just inside the deadline would otherwise go + on to stat, unlink and rmdir every stale manifest entry unbounded — the + opposite of what a small ``timeout`` asks for. + """ + existing = _place_managed(root, "stale", SKILL_BODY) + + report = await write_skills([], root, timeout=0) + + assert report.ok is False + assert existing.exists(), "prune ran past the exhausted deadline" + assert any("timeout was exhausted" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "stale/SKILL.md" in _read_manifest(root)["entries"] + + async def test_a_verification_failure_never_prunes_the_good_copy( + self, root: Path + ) -> None: + """A store may key ``all_objects`` differently from the object's own key. + + The on-disk copy lives under the object's own key, so a failure recorded + under the *store's* dict key would drop the real key out of the + requested set and let prune delete the last known-good copy. + """ + + class AliasKeyedStore: + """Keys objects by an internal id, not by the skill's own key.""" + + def __init__(self, raw: dict[str, Any]) -> None: + self._raw = raw + + def get_object(self, kind: str, key: str) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {"internal-uuid-1": self._raw} + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + tampered = { + "key": "pdf-extraction", + "version": 1, + "content": "tampered\n", + "contentHash": _hash(SKILL_BODY), # does not match the content + } + skills_module._set_store(AliasKeyedStore(tampered)) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert _actions_by_key(report)["pdf-extraction"].action == "error" + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + before = _read_manifest(root) + + await write_skills([SkillReference(key="b", version=1)], root) + + assert ( + _read_manifest(root)["entries"]["a/SKILL.md"] + == (before["entries"]["a/SKILL.md"]) + ) + + +class TestVerifyThenWrite: + """Hash re-verified immediately before writing.""" + + async def test_hash_mismatch_aborts_the_write( + self, root: Path, recording_emitter: Any + ) -> None: + skills_module._set_emitter_for_testing(recording_emitter) + bad = Skill( + key="a", + version=1, + content=SKILL_BODY.encode("utf-8"), + content_hash="0" * 64, + ) + + report = await write_skills([bad], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert not (root / "a" / "SKILL.md").exists() + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_oversize_skill_aborts_the_write(self, root: Path) -> None: + oversize = "x" * (64 * 1024 + 1) + report = await write_skills([_skill("a", 1, oversize)], root) + assert report.ok is False + assert not (root / "a" / "SKILL.md").exists() + + async def test_mismatch_does_not_disturb_existing_managed_file( + self, root: Path + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + bad = Skill(key="a", version=2, content=b"new content\n", content_hash="f" * 64) + + await write_skills([bad], root) + + assert target.read_text(encoding="utf-8") == SKILL_BODY From c88d5bf5ce0db1e8ad9f8fa5a5ee782ea768b887 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 26 Aug 2026 16:26:15 -0400 Subject: [PATCH 07/43] test(client): assert rename containment through the branching helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``test_key_at_the_data_model_bound_is_reported_not_raised`` read ``dst_dir_id`` directly, but that field is only populated when ``os.replace`` is called with ``dir_fd`` kwargs. On the path fallback (``SUPPORTS_DIR_FD`` false — the shape Windows takes) it stays ``None``, so the assertion failed even though the valid skill had been renamed correctly into its own directory. ``_assert_atomic_rename_of`` already branches on both call shapes and asserts the same containment property, plus the single-rename count the list comparison implied. Use it. Verified by forcing the probe off for a whole session: this was the only test in the module that broke under the no-``*at()`` shape, and the helper-based check passes under both. Reported by Cursor Bugbot on #54. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 6502aa5b..5af4d19b 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -932,8 +932,11 @@ async def test_key_at_the_data_model_bound_is_reported_not_raised( # The bare-filename ``dst`` of a ``dir_fd``-relative rename carries no # directory, so "the path does not contain the hostile key" is no longer # a meaningful check. Assert the stronger thing instead: the only rename - # that happened was into the valid skill's own directory. - assert [call.dst_dir_id for call in spy.calls] == [_dir_id(root / "good")] + # that happened was into the valid skill's own directory. Through the + # shared helper, so the check holds on the path fallback too — reading + # ``dst_dir_id`` directly would compare ``None`` there and fail a run + # that had in fact renamed correctly. + _assert_atomic_rename_of(spy, root / "good") # The valid skill is fully reconciled: written AND recorded, not orphaned. assert (root / "good" / "SKILL.md").exists() assert "good/SKILL.md" in _read_manifest(root)["entries"] From 49c2741641051df8b666db78097832b745b2ba6f Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Sun, 30 Aug 2026 04:26:10 -0400 Subject: [PATCH 08/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20self-healing=20reconciles,=20and=20keys=20no=20file?= =?UTF-8?q?system=20can=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps from the Agent Skills security design review, both in `skills_fs.py`: AV-3 (partial reconciles are unrecoverable) and DV-2 (the key grammar admits Windows device names). **AV-3 — adopt a file whose bytes already are the resolved content.** `_write_all` reconciles every skill and only then rewrites the manifest, once, last. A process killed in that window leaves a skill file at a managed path with no manifest entry — exactly the condition `_write_one` treats as an unmanaged-file collision, so the skill was wedged permanently: every later reconcile took the same refusal branch. Boot-time execution under a ten-second budget makes the crash window realistic. `_write_one` now reads and hashes first and decides from the bytes. Content byte-identical to what LaunchDarkly resolved is adopted — manifest entry recorded, reported `skipped_current` — and anything else falls through to the same refusal as before. This cannot weaken the clobber guarantee: differing unmanaged bytes are never overwritten, and the existing clobber tests pass unchanged. Three details carry the safety: - A read that fails is a refusal, never an overwrite, with a message distinguishable from the byte-mismatch refusal — it is the comparison that would otherwise authorize the write, and a file that could not be read has not been shown to be ours. - The read stays on `_read_regular_file`. Adoption widens it to genuinely foreign files, so its refusal of FIFOs and other non-regular files is now load-bearing rather than defensive. It gains a `max_bytes` bound of `len(content) + 1` — enough to prove inequality for anything longer, and what keeps a foreign file of arbitrary size out of memory. A bound of exactly `len(content)` would adopt every file that merely begins with the resolved content. - `skipped_current` is reused rather than adding an `adopted` action kind, so `ReconcileActionKind` — public, and owned by an approved PR — does not change. Its documented meaning already fits. Adoption also makes the file prunable later. That is correct rather than a weakening: only byte-identical LaunchDarkly content is ever adopted, so a later prune removes content LaunchDarkly delivered anyway — what would have happened had the crash not occurred. The review also floats a write-intent journal. Assessed as over-engineered; not built. **AV-3, secondary — sweep orphaned temp files.** `atomic_write` unlinks its temp file on any exception but not after a `SIGKILL`, and `_prune` walks manifest entries, which an orphan never has, so nothing would ever notice one. The second-order effect is worse than the disk: `_prune_one`'s `rmdir` only succeeds on an empty directory, so a single orphan pins a skill's directory forever. The sweep runs on both the write and the prune path, and is the one place this SDK removes a file the manifest does not list, so it is bounded on every axis: inside `//` only, for a key that passes `_key_rejection_reason`; only names `safe_fs` itself recognizes, via a new `is_temp_name` beside the naming code rather than a copy of the format string that could drift from it; only regular files, with the type read off the descriptor; unlinked through the pinned descriptor. It never raises and never aborts a run. **DV-2 — reject the 22 Windows reserved device names.** `con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9` are all valid skill keys and none can be a directory name on Windows. Rejected in `_key_rejection_reason`, which the write and prune paths already share, and *not* in the key grammar: `parse_ai_config` fails closed, so a grammar-level rejection would invalidate an entire AI Config for a Linux customer over a Windows-only constraint, and would silently shrink `skill_refs` — which is what authorizes a prune, turning "fails to write on Windows" into "gets deleted on Linux". The 255-byte component bound is in this layer for the same reason. Unconditional, not platform-gated: a root written from a Linux container is routinely read from a Windows host, and neither repository has a Windows CI runner, so a gated branch would be untestable — the condition that produced the gap. No suffix stripping and no case folding: the grammar admits no `.` and no `$`, so `con.txt` and `CONIN$` are unreachable, and keys are lowercase-only. `com0` and `lpt0` are not reserved and are not included. The trade is real and belongs in the release notes: a customer who legitimately names a skill `aux` now gets a reported `error` action on Linux where it previously worked. Neither gap emits the integrity-failure log record. A key rejection and a clobber refusal are `ReconcileAction` errors, not integrity failures. Tests cover crash-mid-reconcile recovery end to end (adopted, reported, recorded, and the next reconcile an ordinary no-op), byte-differing content still refused untouched, an unmanaged FIFO refused without hanging, a read failure refusing rather than overwriting, the `len + 1` off-by-one, the sweep and the `rmdir` it unblocks, lookalike temp names and a symlink wearing one left alone, all 22 reserved names through both destructive paths, and — the point of the layer choice — each reserved name still valid to `is_valid_skill_key`, `parse_ai_config`, and `skill_refs`. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 26 ++ packages/client/agents.md | 47 ++- .../src/launchdarkly_ai_server/safe_fs.py | 54 ++- .../src/launchdarkly_ai_server/skills_fs.py | 159 ++++++- packages/client/tests/test_skills_fs.py | 388 ++++++++++++++++++ 5 files changed, 655 insertions(+), 19 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index d9c1db15..711ce5bf 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -357,6 +357,32 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed +path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is +adopted — recorded in the manifest and reported `skipped_current` — rather than refused. +Without that, a process killed after a skill file lands but before the manifest is rewritten +leaves that file managed-but-unrecorded, which is indistinguishable from a file you wrote +yourself, so every later reconcile would refuse it and the skill would stay wedged until +someone intervened. Adoption cannot weaken the guarantee above, because bytes that differ in +any way are still refused and left untouched. Note that an adopted file becomes prunable +like any other managed file — which is the same outcome the crash pre-empted. + +**A few keys are legal to an AI Config but not to a filesystem.** A key becomes a single +directory name, so `write_skills` applies bounds of its own on top of the key grammar: no +mainstream filesystem allows a 256-byte path component, and Windows reserves 22 MS-DOS +device names (`con`, `prn`, `aux`, `nul`, `com1`–`com9`, `lpt1`–`lpt9`) that cannot be +directory names there. Either one is a reported `error` action for that skill, and the +rejection is unconditional rather than platform-gated — a managed root written from a Linux +container is routinely read from a Windows host, so the on-disk result must not depend on +which OS ran the write. The keys stay valid everywhere else: an AI Config referencing a skill +named `aux` parses, and its other fields are unaffected. If you have a skill named for a +device, rename it. + +**Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per +*component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed +Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on +Windows. + | Export | Description | |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | diff --git a/packages/client/agents.md b/packages/client/agents.md index feb2e348..cd99b28f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -250,7 +250,26 @@ Store data is **untrusted input**; the transport is not part of the trust bounda - **Never write through a symlink**, in either the skill directory or the target file, on the write path *and* the prune path. - **Destructive operations only on manifest-listed paths whose `key` matches.** A file at a - managed path with no matching manifest entry is reported as `error` and left alone. + managed path with no matching manifest entry is reported as `error` and left alone — + *unless its bytes already are the resolved content*, in which case it is adopted (manifest + entry recorded, reported `skipped_current`). That single exception is what makes a + reconcile killed between the content writes and the final manifest rewrite recoverable + instead of permanently wedged, and it cannot be widened: the comparison is over the + verbatim bytes against the resolved `contentHash`, a read that fails is a refusal and + never an overwrite, and the read is bounded at `len(content) + 1` bytes so a file that + merely *begins* with the resolved content is refused too. Do not relax it to a prefix, a + length, an mtime, or the manifest's own recorded `sha256` — that field is untrusted and is + never a decision input. `skipped_current` is reused deliberately rather than adding an + `adopted` action kind; `ReconcileActionKind` is a public closed set. +- **Temp files are swept, within the same bounds as everything else.** `atomic_write` unlinks + its own temp file on any exception, but a `SIGKILL` leaves one behind that no manifest + entry records, and a non-empty directory defeats `_prune_one`'s `rmdir` — so one orphan + pins a skill directory forever. The sweep is the only place this SDK removes a file the + manifest does not list, and it is bounded on every axis: inside `//` only, for a + key that passes `_key_rejection_reason`; only names `safe_fs.is_temp_name` recognizes, + anchored at both ends and asked of `safe_fs` rather than re-spelled (a copy would drift + from the writer); only regular files, with the type read off the descriptor; unlinked + through the pinned descriptor. It never raises and never aborts a run. - **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed `entries`, or a `manifestVersion` this release cannot read means no overwrites and no prunes, brand-new paths may still be written, an `error` action names the manifest, and @@ -265,10 +284,28 @@ Store data is **untrusted input**; the transport is not part of the trust bounda "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at write or unlink time reopens a swap window that the checks above cannot cover. - **A key valid to the data model may still be unrepresentable on disk.** The model allows - 256 characters; `NAME_MAX` is 255 bytes. `write_skills` rejects an over-long key before - any filesystem call, and every per-skill filesystem failure is caught at the loop so it - becomes an `error` action — aborting the loop would skip the manifest rewrite and orphan - files already written in that run. + 256 characters; `NAME_MAX` is 255 bytes. Windows additionally reserves 22 MS-DOS device + names, none of which can be a directory name there: `con`, `prn`, `aux`, `nul`, + `com1`–`com9`, `lpt1`–`lpt9` (`com0` and `lpt0` are *not* reserved; do not add them). + `write_skills` rejects both before any filesystem call, and every per-skill filesystem + failure is caught at the loop so it becomes an `error` action — aborting the loop would + skip the manifest rewrite and orphan files already written in that run. +- **Those two bounds live in `_key_rejection_reason`, not in the key grammar, and must not + move.** `is_valid_skill_key` / `skill_key_rejection_reason` keep admitting an over-long or + reserved key on purpose. `parse_ai_config` fails closed on a bad `skills` entry, so a + grammar-level rejection would invalidate the *entire* AI Config — model, provider, + instructions, tools — for a Linux customer over a Windows-only constraint; and it would + silently shrink `skill_refs`, which is what authorizes a prune, converting "this skill + fails to write on Windows" into "this skill gets deleted on Linux". `_key_rejection_reason` + is shared by the write and prune paths, so one edit covers both destructive paths. + The reserved-name check is unconditional rather than `os.name == "nt"`-gated: a root + written from a Linux container is routinely read from a Windows host, and neither + repository has a Windows CI runner (every matrix job is `ubuntu-latest`), so a gated branch + would be untestable — the exact condition that produced the gap. No suffix stripping and no + case folding are needed, because the grammar admits no `.` and no `$` (so `con.txt` and + `CONIN$` are unreachable) and is lowercase-only. The residual the SDK cannot check is total + path length: the 255-byte bound is per *component*, and the root belongs to the customer, + so `MAX_PATH` overflow is a README note rather than a check. - **A key is untrusted input everywhere it appears.** `skill_key_rejection_reason` is the single canonical explanation, so the config parser and the reference projection reject a key for the same stated reason — and so does every layer added later. A silently diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 3605399e..73eef878 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -17,6 +17,7 @@ import errno import os +import re import secrets import stat import tempfile @@ -191,6 +192,53 @@ def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: os.unlink(name, dir_fd=dir_fd) +_TEMP_SUFFIX = ".tmp" +"""Suffix on every temp file this module creates.""" + +_TEMP_TOKEN_BYTES = 8 +"""Bytes of randomness in a temp name, as ``secrets.token_hex`` takes them.""" + +_TEMP_TOKEN_PATTERN = re.compile( + # Two producers, one recognizer. The descriptor path below names its temp + # file with ``secrets.token_hex(_TEMP_TOKEN_BYTES)`` — twice that many + # lowercase hex characters. The fallback path hands naming to + # ``tempfile.mkstemp``, whose sequence is eight characters drawn from + # ``[a-z0-9_]``. Matched with ``fullmatch``, which anchors both branches at + # both ends, so nothing longer or otherwise-shaped is ever recognized. + rf"[0-9a-f]{{{_TEMP_TOKEN_BYTES * 2}}}|[a-z0-9_]{{8}}" +) + + +def temp_name_prefix(name: str) -> str: + """ + The prefix every temp file for *name* is created under. + + Spelled once because two callers need to agree on it: ``atomic_write`` + creates the name, and a caller sweeping orphaned temp files left by a crash + has to recognize it. A copy of the format string in the sweeper would be a + copy that can drift out of step with the writer. + """ + return f".{name}." + + +def is_temp_name(candidate: str, name: str) -> bool: + """ + Whether *candidate* is a name this module could have created for *name*. + + The recognizer for the orphan sweep: ``atomic_write`` unlinks its temp file + on any exception, but a ``SIGKILL`` between the create and the rename leaves + it behind, and nothing else on disk records that it exists. Deliberately + narrow — prefix, random token, and suffix must all match, with nothing + before or after — because the only thing a caller does with a ``True`` here + is delete the file. + """ + prefix = temp_name_prefix(name) + if not candidate.startswith(prefix) or not candidate.endswith(_TEMP_SUFFIX): + return False + token = candidate[len(prefix) : -len(_TEMP_SUFFIX)] + return _TEMP_TOKEN_PATTERN.fullmatch(token) is not None + + def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ ``tempfile.mkstemp`` for a directory descriptor. @@ -202,7 +250,7 @@ def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: """ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) for _ in range(tempfile.TMP_MAX): - name = f"{prefix}{secrets.token_hex(8)}.tmp" + name = f"{prefix}{secrets.token_hex(_TEMP_TOKEN_BYTES)}{_TEMP_SUFFIX}" try: return os.open(name, flags, 0o600, dir_fd=dir_fd), name except FileExistsError: @@ -234,7 +282,7 @@ def atomic_write( semantics on Windows). """ at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None - prefix = f".{name}." + prefix = temp_name_prefix(name) target: str | Path if at_fd is not None: @@ -243,7 +291,7 @@ def atomic_write( else: # mkstemp opens with O_CREAT|O_EXCL, so an existing temp path is never # reused. - fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=".tmp") + fd, temp = tempfile.mkstemp(dir=directory, prefix=prefix, suffix=_TEMP_SUFFIX) target = directory / name try: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index feb8fbc9..8a865dd8 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -33,6 +33,7 @@ SymlinkRefused, atomic_write, atomic_write_in, + is_temp_name, pinned_directory, unlink_file, ) @@ -95,6 +96,27 @@ """ +_WINDOWS_RESERVED_NAMES = frozenset( + {"con", "prn", "aux", "nul"} + | {f"com{digit}" for digit in range(1, 10)} + | {f"lpt{digit}" for digit in range(1, 10)} +) +""" +The 22 MS-DOS device names Windows still reserves, which cannot be directory +names there. The key grammar admits every one of them, so a customer who names a +skill ``con`` gets a working reconcile on Linux and a broken one on Windows — +rejected here instead, on every platform, so the on-disk result never depends on +which OS ran the write. Neither repository has a Windows CI runner, which is the +condition that produced the gap in the first place. + +The bare names are the whole set: no suffix stripping is needed because the key +grammar admits no ``.``, so ``con.txt`` is unreachable, and ``CONIN$`` / +``CONOUT$`` are unreachable for want of a ``$``; no case folding is needed +because the grammar is lowercase-only. ``com0`` and ``lpt0`` are deliberately +absent — those are not reserved. +""" + + # ------------------------------------------------------------------------- # The reconcile entry point # ------------------------------------------------------------------------- @@ -644,6 +666,17 @@ def _key_rejection_reason(key: Any) -> str | None: f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" ) + # Same reasoning as the byte bound above, and it lives at the same layer for + # the same reason: the grammar itself must keep admitting these, because + # rejecting them there would fail the whole AI Config over one skill, and + # would shrink ``skill_refs`` — which is what authorizes a prune, so a + # Windows-only constraint would delete the skill's file on Linux. + if key in _WINDOWS_RESERVED_NAMES: + return ( + f"skill key '{key}' is a name Windows reserves for a device and " + "cannot be a directory name there; it is rejected on every platform " + "so a managed root written on one OS is usable on the other" + ) return None @@ -683,24 +716,45 @@ def failed(message: str) -> ReconcileAction: ) encoded, content_hash = verified.encoded, verified.content_hash + # Sweep before writing rather than after, so a temp file this run is about + # to create can never be a candidate. + _sweep_orphan_temp_files(root, key) + # Overwrite only what the manifest records as ours under this key. entry = entries.get(relative) managed = isinstance(entry, dict) and entry.get("key") == key exists = target.exists() - if exists and not managed: - return failed( - f"'{relative}' exists but the manifest does not record it as managed " - f"under key '{key}'; refusing to overwrite a file this SDK did not write" - ) - if exists: + # Hash first, and decide from the bytes. The manifest check below is what + # protects a customer's own file, but it also refuses the file this SDK + # itself wrote and was killed before recording — the reconcile writes + # every skill and only then rewrites the manifest, so a crash in that + # window leaves a managed path with no entry, and every later reconcile + # takes the refusal branch forever. Comparing the bytes distinguishes the + # two cases without weakening anything: only content byte-identical to + # what LaunchDarkly resolved is ever adopted. try: - on_disk = _read_regular_file(target) + on_disk = _read_regular_file(target, max_bytes=len(encoded)) except OSError as exc: + if not managed: + # A read that failed proves nothing, and must never become an + # overwrite: it is the comparison below that would authorize one. + return failed( + f"'{relative}' exists, the manifest does not record it as " + f"managed under key '{key}', and it could not be read to " + f"compare against the resolved content: {exc}; refusing to " + "overwrite a file this SDK may not have written" + ) return failed(f"'{relative}' could not be read: {exc}") if hashlib.sha256(on_disk).hexdigest() == content_hash: + # ``skipped_current`` covers this deliberately, rather than a new + # action kind: its documented meaning is that the bytes on disk + # already are the resolved content, which is exactly as true for an + # adopted file as for one this SDK wrote and recorded. Adoption does + # add a manifest entry, so the file becomes prunable later — correct, + # because a prune then removes content LaunchDarkly delivered anyway. _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, "skipped_current") return ReconcileAction( @@ -709,6 +763,12 @@ def failed(message: str) -> ReconcileAction: version=skill.version, path=str(target), ) + + if not managed: + return failed( + f"'{relative}' exists but the manifest does not record it as managed " + f"under key '{key}'; refusing to overwrite a file this SDK did not write" + ) # Stale version or local tampering — LD-resolved content wins. action: ReconcileActionKind = "updated" else: @@ -725,7 +785,7 @@ def failed(message: str) -> ReconcileAction: ) -def _read_regular_file(target: Path) -> bytes: +def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: """ Reads *target*, refusing anything that is not a regular file. @@ -739,6 +799,12 @@ def _read_regular_file(target: Path) -> bytes: bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor without it translates CRLF on read, which would fail the hash comparison against content that is actually current. + + Reads at most ``max_bytes + 1`` bytes. The only consumer compares a hash, and + anything longer than the resolved content cannot match it, so the one extra + byte is enough to prove inequality — which is what keeps a foreign file of + arbitrary size from being pulled into memory now that adoption reads files + the manifest does not list. """ flags = ( os.O_RDONLY @@ -751,15 +817,82 @@ def _read_regular_file(target: Path) -> bytes: if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError("the target file is not a regular file") chunks: list[bytes] = [] - while True: - chunk = os.read(fd, 65536) + remaining = max_bytes + 1 + while remaining > 0: + chunk = os.read(fd, min(remaining, 65536)) if not chunk: - return b"".join(chunks) + break chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) finally: os.close(fd) +def _sweep_orphan_temp_files(root: Path, key: str) -> None: + """ + Removes temp files a killed reconcile left behind under ``//``. + + ``atomic_write`` unlinks its own temp file on any exception, but a ``SIGKILL`` + between the create and the rename leaves one on disk, and nothing else + records that it exists: ``_prune`` walks manifest entries, and an orphan + never has one. The second-order effect is what makes this worth doing — + ``_prune_one``'s ``rmdir`` only succeeds on an empty directory, so a single + orphaned temp pins a skill's directory permanently. + + Bounded on every axis, because this is the one place the SDK removes a file + the manifest does not list: only inside a directory named by a key that + passes ``_key_rejection_reason``; only names ``safe_fs`` itself recognizes as + its own temp naming for ``SKILL.md``, anchored at both ends, and asked of + ``safe_fs`` rather than re-spelled here so the recognizer cannot drift from + the writer; only regular files; and every removal relative to a descriptor + pinned with ``O_NOFOLLOW``. It never raises and never aborts the run: the + reconcile itself has succeeded either way, so a sweep that cannot happen is + a warning. + """ + if _key_rejection_reason(key) is not None: + return + skill_dir = root / key + if not skill_dir.is_dir(): + return + + try: + with pinned_directory(skill_dir) as dir_fd: + # Listing by path is safe even though the removals are + # descriptor-relative: a name reaches the unlink only if it matches + # the anchored temp pattern, and the unlink resolves it inside the + # pinned directory, so a listing redirected between the pin and here + # can at worst name a file that is not in it. + for name in sorted(os.listdir(skill_dir)): + if is_temp_name(name, SKILL_FILENAME): + _remove_orphan_temp_file(skill_dir, name, dir_fd) + except (OSError, ValueError) as exc: + logger.warning( + "orphaned temp files under skill '%s' could not be swept: %s", key, exc + ) + + +def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> None: + """ + Removes one recognized orphan. A per-file failure warns and moves on. + + The type check is what keeps the temp naming from being a way to have this + SDK delete something it did not write: a symlink or a FIFO wearing that name + is not a file ``atomic_write`` left behind, so it is not this function's to + remove. It is read off the descriptor, not the path, wherever there is one. + """ + try: + if dir_fd is not None: + mode = os.stat(name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(skill_dir / name).st_mode + if not stat.S_ISREG(mode): + return + unlink_file(skill_dir, name, dir_fd=dir_fd) + except (OSError, ValueError) as exc: + logger.warning("an orphaned temp file could not be removed: %s", exc) + + def _write_through_descriptor( skill_dir: Path, encoded: bytes, key: str, relative: str ) -> str | None: @@ -929,6 +1062,10 @@ def _prune_one( if unsafe is not None: return _prune_error(key, f"'{relative}' was not removed: {unsafe}", version) + # Before the removal, so the ``rmdir`` below is not defeated by an orphaned + # temp file that nothing else on disk records. + _sweep_orphan_temp_files(root, key) + removed_from_disk = False if target.exists(): failure = _unlink_through_descriptor(skill_dir, relative) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 5af4d19b..223a3262 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -26,8 +26,11 @@ SkillReference, get_skill, init_client, + parse_ai_config, + skill_refs, write_skills, ) +from launchdarkly_ai_server.types_validation import is_valid_skill_key MANIFEST_NAME = ".launchdarkly-skills.json" SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" @@ -1640,3 +1643,388 @@ async def test_integrity_signal_property_keys_match_across_layers( f"write-only keys: {sorted(write_keys - accessor_keys)}" ) assert "expected_hash" in accessor_keys + + +# --------------------------------------------------------------------------- +# Self-healing partial reconciles +# --------------------------------------------------------------------------- + + +def _place_unmanaged(root: Path, key: str, content: str) -> Path: + """A file at a managed path with **no** manifest entry. + + Exactly the state a reconcile killed between the content writes and the + final manifest rewrite leaves behind — and, indistinguishably on disk, the + state a customer authoring their own file there creates. Which is why the + bytes are the only thing that may decide between them. + """ + target = root / key / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + +class TestCrashMidReconcileRecovery: + """A crash between the writes and the manifest rewrite must not wedge a skill.""" + + async def test_byte_identical_unmanaged_file_is_adopted(self, root: Path) -> None: + """The whole point: the second reconcile repairs the first one's crash. + + Without adoption every later reconcile takes the clobber-refusal branch + forever, because the file is at a managed path with no manifest entry. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + + first = await write_skills([_skill("a")], root) + + assert first.ok is True, _error_messages(first) + action = _actions_by_key(first)["a"] + assert action.action == "skipped_current" + assert action.version == 1 + assert action.path == str(target) + # Adopted, not rewritten, and now recorded. + assert target.read_text(encoding="utf-8") == SKILL_BODY + entry = _read_manifest(root)["entries"]["a/SKILL.md"] + assert entry["key"] == "a" + assert entry["version"] == 1 + assert entry["sha256"] == _hash(SKILL_BODY) + + # And the run after it is an ordinary no-op, through the managed path. + second = await write_skills([_skill("a")], root) + assert second.ok is True, _error_messages(second) + assert _actions_by_key(second)["a"].action == "skipped_current" + + async def test_adoption_writes_nothing( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is a manifest edit, not a write. Nothing touches the bytes.""" + _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert spy.calls == [] + + async def test_adoption_reports_skipped_current_not_a_new_action_kind( + self, root: Path, recording_emitter: Any + ) -> None: + """``skipped_current`` is reused deliberately — no ``adopted`` kind exists.""" + skills_module._set_emitter_for_testing(recording_emitter) + _place_unmanaged(root, "a", SKILL_BODY) + + report = await write_skills([_skill("a")], root) + + assert {a.action for a in report.actions} == {"skipped_current"} + props = recording_emitter.signals(MATERIALIZED_SIGNAL)[0] + assert props["reconcile_action"] == "skipped_current" + assert props["skill_key"] == "a" + + async def test_an_adopted_file_is_prunable_afterwards(self, root: Path) -> None: + """The documented caveat, pinned. + + Adoption records a manifest entry, so a later reconcile may prune the + file. That is correct rather than a weakening: only content byte-identical + to what LaunchDarkly resolved is ever adopted, so the prune removes + content LaunchDarkly delivered — exactly what would have happened had the + crash never occurred. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + await write_skills([_skill("a")], root) + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not target.exists() + + async def test_differing_unmanaged_content_is_still_refused( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The clobber guarantee, restated against the adoption rule. + + Adoption compares bytes, so anything that is not byte-identical to the + resolved content falls through to the same refusal as before. + """ + target = _place_unmanaged(root, "a", "user authored\n") + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "did not write" in action.error + assert target.read_text(encoding="utf-8") == "user authored\n" + assert spy.calls == [] + + async def test_a_longer_file_sharing_the_content_prefix_is_not_adopted( + self, root: Path + ) -> None: + """The read is bounded at ``len(content) + 1``, and that one byte matters. + + A bound of exactly ``len(content)`` would make every file that merely + *begins* with the resolved content hash as current, adopting — and later + pruning — a customer file with the skill body at its head. + """ + longer = SKILL_BODY + "and my own notes below\n" + target = _place_unmanaged(root, "a", longer) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "error" + assert target.read_text(encoding="utf-8") == longer + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="no FIFOs on this platform") + async def test_an_unmanaged_fifo_is_refused_and_never_read( + self, root: Path + ) -> None: + """Adoption widened the read to foreign files, so this refusal is load-bearing. + + Opening a FIFO with no writer blocks forever; the descriptor-pinned read + opens ``O_NONBLOCK`` and rejects anything that is not a regular file, so + this returns rather than hanging the reconcile and the event loop with it. + """ + skill_dir = root / "a" + skill_dir.mkdir() + os.mkfifo(skill_dir / "SKILL.md") + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "regular file" in action.error + assert stat.S_ISFIFO(os.lstat(skill_dir / "SKILL.md").st_mode) + + async def test_a_read_failure_on_an_unmanaged_file_refuses( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A failed read proves nothing, so it must never become an overwrite. + + The adoption comparison is what would otherwise authorize the write, and + a file whose bytes could not be read has not been shown to be ours. + """ + target = _place_unmanaged(root, "a", SKILL_BODY) + spy = _ReplaceSpy().install(monkeypatch) + real_open = os.open + + def refuse_the_target(path: Any, *args: Any, **kwargs: Any) -> int: + if isinstance(path, (str, os.PathLike)) and os.fspath(path) == str(target): + raise PermissionError(13, "Permission denied") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(skills_fs_module.os, "open", refuse_the_target) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # Distinguishable from the byte-mismatch refusal: this one says the + # comparison could not be made at all. + assert "could not be read to compare" in action.error + assert spy.calls == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "a/SKILL.md" not in _read_manifest(root)["entries"] + + +# --------------------------------------------------------------------------- +# Orphaned temp files +# --------------------------------------------------------------------------- + + +def _temp_name(token: str = "0123456789abcdef") -> str: + """A name ``atomic_write`` could have created for ``SKILL.md``. + + The prefix comes from ``safe_fs`` itself rather than a copy of its format + string, so a change to the naming breaks this helper instead of silently + making the sweep a no-op. + """ + return f"{safe_fs_module.temp_name_prefix('SKILL.md')}{token}.tmp" + + +class TestOrphanedTempFiles: + """A ``SIGKILL`` mid-write leaves a temp file nothing else records.""" + + async def test_an_orphan_is_swept_on_the_next_write(self, root: Path) -> None: + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert not orphan.exists() + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_an_orphan_no_longer_blocks_directory_cleanup( + self, root: Path + ) -> None: + """The second-order effect: ``rmdir`` fails on a non-empty directory. + + One orphaned temp file would otherwise pin the skill's directory under + the managed root forever, long after the skill itself was revoked. + """ + _place_managed(root, "a", SKILL_BODY) + orphan = root / "a" / _temp_name() + orphan.write_text("half-written body", encoding="utf-8") + + report = await write_skills([], root) + + assert report.ok is True, _error_messages(report) + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + @pytest.mark.parametrize( + "innocent", + [ + "notes.tmp", + "SKILL.md.tmp", + ".SKILL.md.tmp", + ".SKILL.md..tmp", + _temp_name("not-a-token"), + _temp_name("0123456789abcdef") + ".bak", + "x" + _temp_name(), + _temp_name("0123456789abcdefff"), + ], + ) + async def test_a_lookalike_name_is_left_alone( + self, root: Path, innocent: str + ) -> None: + """The recognizer is anchored at both ends, and the sweep deletes files. + + Anything that is not exactly the naming ``safe_fs`` produces belongs to + the customer, whatever it resembles. + """ + _place_managed(root, "a", SKILL_BODY) + bystander = root / "a" / innocent + bystander.write_text("mine\n", encoding="utf-8") + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert bystander.read_text(encoding="utf-8") == "mine\n" + + @pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" + ) + async def test_a_symlink_wearing_the_temp_name_is_not_removed( + self, root: Path, tmp_path: Path + ) -> None: + """The temp naming must not become a way to have the SDK delete elsewhere. + + Only a regular file is ever swept, and the type comes off the descriptor + rather than a followed path. + """ + outside = tmp_path / "precious.txt" + outside.write_text("do not delete\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + link = root / "a" / _temp_name() + link.symlink_to(outside) + + report = await write_skills([_skill("a")], root) + + assert report.ok is True, _error_messages(report) + assert outside.read_text(encoding="utf-8") == "do not delete\n" + assert link.is_symlink() + + +# --------------------------------------------------------------------------- +# Windows reserved device names +# --------------------------------------------------------------------------- + +# Spelled out independently of the implementation's own set, so a name dropped +# from that set fails here rather than agreeing with itself. +WINDOWS_RESERVED_KEYS = ( + ["con", "prn", "aux", "nul"] + + [f"com{digit}" for digit in range(1, 10)] + + [f"lpt{digit}" for digit in range(1, 10)] +) + + +class TestWindowsReservedNames: + """Keys Windows cannot hold as directory names, refused on every platform.""" + + def test_the_set_is_exactly_twenty_two_names(self) -> None: + assert len(WINDOWS_RESERVED_KEYS) == len(set(WINDOWS_RESERVED_KEYS)) == 22 + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_write_skills( + self, root: Path, reserved: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _ReplaceSpy().install(monkeypatch) + + report = await write_skills([_skill(reserved)], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "Windows reserves" in action.error + assert reserved in action.error + # Rejected before any filesystem call, not by the OS. + assert spy.calls == [] + assert not (root / reserved).exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + async def test_a_reserved_key_is_refused_by_the_prune_path( + self, root: Path, reserved: str + ) -> None: + """``_key_rejection_reason`` gates both destructive paths, so both refuse. + + A manifest naming a reserved key is left in place rather than acted on: + the same key check that stops the write stops the delete. + """ + target = _place_managed(root, reserved, SKILL_BODY) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)[reserved] + assert action.action == "error" + assert action.error is not None + assert "left in place" in action.error + assert target.read_text(encoding="utf-8") == SKILL_BODY + + @pytest.mark.parametrize("not_reserved", ["com0", "lpt0", "con1", "nul2", "conx"]) + async def test_neighbouring_names_are_not_reserved( + self, root: Path, not_reserved: str + ) -> None: + """``com0`` and ``lpt0`` are not device names, and must still write.""" + report = await write_skills([_skill(not_reserved)], root) + + assert report.ok is True, _error_messages(report) + assert (root / not_reserved / "SKILL.md").exists() + + @pytest.mark.parametrize("reserved", WINDOWS_RESERVED_KEYS) + def test_a_reserved_name_is_still_a_valid_key_to_every_pure_layer( + self, reserved: str + ) -> None: + """The layer choice, asserted — this is the whole point of it. + + The constraint lives in the filesystem layer and must not migrate into + the key grammar. At the grammar level a rejection would fail the *entire* + AI Config — model, provider, instructions, tools — for a Linux customer + over a Windows-only constraint, and would shrink ``skill_refs``, which is + what authorizes a prune: "this skill fails to write on Windows" would + become "this skill gets deleted on Linux". + """ + assert is_valid_skill_key(reserved) is True + + parsed = parse_ai_config( + { + "model": {"name": "claude-3"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + "skills": [{"key": reserved, "version": 1}], + } + ) + assert parsed.success is True + + refs = skill_refs(parsed.data) + assert [ref.key for ref in refs] == [reserved] From eda4ae1f9219a8a3c3b363dcfc05fee566820825 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 31 Aug 2026 14:04:18 -0400 Subject: [PATCH 09/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20a=20distinguishable=20outcome=20for=20integrity=20f?= =?UTF-8?q?ailure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``get_skill`` returns ``None`` for four unrelated outcomes: no such skill, the store raised, the requested version is not the one held, and content that failed hash verification. A caller cannot fail closed on suspected tampering while tolerating a merely-absent skill, so no automated customer-side response is possible — finding LA-2 of the Agent Skills security design review. The information already existed internally, as prose in ``Resolution.error``. This gives it a token: ``Resolution`` grows a typed ``reason``, set explicitly at every construction site and declared without a default so a sixth outcome added later has to choose which public token it maps to. ``get_skill_result`` maps that straight through to a frozen ``SkillOutcome`` (``skill``, ``reason``, ``detail``). Deriving the public reason by matching the error string is the fragility LA-2 is about, so the mapping is readable in one table. ``get_skill`` is untouched — its ``None``-for-every-failure contract is documented in its docstring and in the README, and a test now pins that all four failures still collapse to ``None`` and still never raise. Nothing new is emitted: Gap 1's integrity record already fired inside verification before ``resolve_from_store`` returned, and a test asserts one failed retrieval still produces exactly one record and one signal. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 56 +++- packages/client/agents.md | 59 +++- .../src/launchdarkly_ai_server/__init__.py | 8 +- .../src/launchdarkly_ai_server/skills.py | 36 ++- .../src/launchdarkly_ai_server/skills_core.py | 39 ++- .../src/launchdarkly_ai_server/skills_fs.py | 10 +- .../src/launchdarkly_ai_server/types.py | 55 ++++ packages/client/tests/test_skills.py | 281 ++++++++++++++++++ 8 files changed, 527 insertions(+), 17 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 711ce5bf..045180b5 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -337,6 +337,55 @@ truncated payload; a mismatch means content was delivered whose bytes are not th LaunchDarkly hashed, which is a possible **active-tampering** signal. Alert on it, and treat `expected_hash` / `observed_hash` as the evidence pair. +#### Failing closed on tampering + +The log record above is the operator's surface. `get_skill_result` is the application's: +same retrieval, same verification, same telemetry as `get_skill`, and it reports which of +five outcomes happened instead of collapsing all of them to `None`. + +```python +from launchdarkly_ai_server import get_skill_result + +outcome = await get_skill_result("pdf-extraction") + +if outcome.reason == "integrity_failure": + # Content was delivered and did not verify. Do not degrade quietly. + raise SystemExit(f"refusing to start: {outcome.detail}") + +if outcome.reason == "store_unavailable": + # The store could not answer at all. Retry, alert, or carry on with what + # you already have — but this is an outage, not a revocation. + print(f"skill retrieval unavailable: {outcome.detail}") +elif outcome.reason in ("absent", "wrong_version"): + # Nothing was tampered with — this skill is simply not available to you. + print(f"continuing without a skill: {outcome.detail}") +elif outcome.skill is not None: + print(outcome.skill.content) +``` + +| `reason` | Meaning | +|---|---| +| `ok` | A verified skill was returned; `.skill` is set and `.detail` is `None`. | +| `absent` | The store answered, and does not hold that key. | +| `integrity_failure` | Content was delivered and failed verification, so it was withheld. **The one to fail closed on.** | +| `store_unavailable` | The store itself could not answer — it raised. An outage, not a deletion. | +| `wrong_version` | The store answered with a version other than the one asked for, so the answer was withheld. | + +`.detail` is human-readable and safe to log or show an operator — it names the key and the +failure mode, and never carries skill content or a filesystem path. Branch on `.reason`, +not on `.detail`. `.skill` is populated only when `.reason == "ok"`. `SkillOutcome` is +frozen, like every other value type here. + +**`get_skill` is unchanged.** It still returns `None` for all four failures and still never +raises for one, so no existing caller has to move. The two accessors run the same code path +and differ only in what they report — `get_skill_result` adds no second log record and no +second signal for a failure that already emitted one, so a caller can switch to it without +double-counting anything. + +`get_skills` and `all_skills` have no reported form: they still omit entries that could not +be resolved, and a run that omitted anything logs a count at WARN. Retrieve individually +with `get_skill_result` when you need the reason per key. + **Versions are selected, not filtered.** A store may hold several versions of one key at once, because a delivery payload does: the newest version of every skill, plus every version a variation currently pins. `get_skill("k", version=1)` asks the store for version @@ -387,6 +436,7 @@ Windows. |---|---| | `skill_refs(config)` | Project a config's `skills` array into `list[SkillReference]`. Pure — no client, store, or network needed. Returns `[]` when absent. | | `get_skill(key, *, version=None)` | One verified skill, or `None`. `version=None` means newest available; a specific `version` matches exactly. Raises only when no store is configured. | +| `get_skill_result(key, *, version=None)` | The same retrieval, reporting **why**: a frozen `SkillOutcome` with `.skill`, `.reason` (`ok` / `absent` / `integrity_failure` / `store_unavailable` / `wrong_version`), and `.detail`. Use it to fail closed on tampering — see *Failing closed on tampering* above. Raises only when no store is configured. | | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | @@ -406,9 +456,10 @@ manifest, for instance — carries the empty string as its `key`. The fixed on-disk values are exported too, so you do not have to hardcode them: `MANIFEST_FILENAME` (`.launchdarkly-skills.json`, handy for a `.gitignore`), -`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the two closed-set types, for annotating +`SKILL_FILENAME`, and `MANIFEST_VERSION`. So are the three closed-set types, for annotating your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_current` / -`removed` / `error`) and `OnUnavailable` (`keep` / `raise`). +`removed` / `error`), `OnUnavailable` (`keep` / `raise`), and `SkillOutcomeReason` +(`absent` / `integrity_failure` / `ok` / `store_unavailable` / `wrong_version`). **`write_skills` blocks.** It is `async` for parity with the other accessors and with the TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, @@ -460,5 +511,6 @@ All types are exported from this package. Handler packages import them from here | `GraphTopology` | The parsed graph flag shape (`root` + `edges`) | | `Skill` | A frozen skill document: `.key`, `.version`, `.content` (verified verbatim `bytes`), `.content_hash`, `.name?`, `.description?` | | `SkillReference` | A frozen version-pinned pointer to a skill: `.key`, `.version` | +| `SkillOutcome` | A frozen retrieval outcome: `.skill`, `.reason` (`SkillOutcomeReason`), `.detail` | | `ReconcileAction` | One `write_skills` outcome: `.key`, `.action`, `.version?`, `.path?`, `.error?` | | `ReconcileReport` | The `write_skills` result: `.actions`, `.ok`, and `.errors` | diff --git a/packages/client/agents.md b/packages/client/agents.md index cd99b28f..a7b4ce1b 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -77,10 +77,10 @@ from launchdarkly_ai_server import config, graph, resolve_graph # Agent Skills from launchdarkly_ai_server import ( - skill_refs, get_skill, get_skills, all_skills, write_skills, - SkillStore, InMemorySkillStore, + skill_refs, get_skill, get_skill_result, get_skills, all_skills, write_skills, + SkillStore, InMemorySkillStore, SkillOutcome, SKILL_FILENAME, MANIFEST_FILENAME, MANIFEST_VERSION, - ReconcileActionKind, OnUnavailable, # the two closed-set unions + ReconcileActionKind, OnUnavailable, SkillOutcomeReason, # the three closed-set unions ) ``` @@ -188,8 +188,8 @@ Three layers, in increasing order of blast radius: typed `SkillReference` values. Pure: no network, no client, no store, no telemetry. Validation of the array itself lives in `parse_ai_config` and is **fail closed** — one malformed reference fails the whole config parse. -2. **Content accessors** — `get_skill`, `get_skills`, `all_skills` read through the - `SkillStore` seam. Configure a store with +2. **Content accessors** — `get_skill`, `get_skill_result`, `get_skills`, `all_skills` read + through the `SkillStore` seam. Configure a store with `init_client(options={"skillStore": store})`; with none configured the accessors raise an actionable `RuntimeError`. A delivery transport can be added behind the seam without touching the public API. @@ -217,6 +217,55 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The reported outcome vocabulary, and the `Resolution` mapping + +`get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` +(`skill`, `reason`, `detail`) naming *which* outcome happened. Both are +`resolve_from_store` — one retrieval, one verification, one telemetry pass — and they differ +only in what they report. `get_skill`'s contract is load-bearing and **frozen**: `None` for +every failure, never raises for one, documented in its docstring and in the README. Change +it and every caller that treats `None` as "no skill" breaks silently. + +`SkillOutcomeReason` is five tokens, listed alphabetically for the same reason +`IntegrityReasonCode` is — so the vocabulary reads identically in the Python and TypeScript +SDKs, where the type name, the accessor name, and the tokens are all deliberately the same. +Do not rename one on one side. + +Internal `Resolution.reason` maps 1:1 onto it, set explicitly at every construction site: + +| `resolve_from_store` outcome | `reason` | +|---|---| +| the store raised (`unavailable=True`) | `store_unavailable` | +| `raw` is not a dict | `absent` | +| `verify_raw_skill` returned `None` | `integrity_failure` | +| `skill.version != wanted_version` | `wrong_version` | +| success | `ok` | + +**Adding a sixth internal outcome means choosing which public token it maps to.** +`Resolution.reason` has no default, so the compiler asks the question; answer it rather than +defaulting to `absent`, which claims the store does not hold the skill. If the new outcome +is genuinely neither of the five, the token set grows — on both sides, in the same commit. + +Two things the reason is deliberately *not*: + +- **Not derived from `Resolution.error`.** That string is prose for a human; recovering a + decision a caller fails closed on by matching it is the fragility the typed token exists + to remove. `detail` *is* that string, passed straight through — safe to surface (key and + failure mode only, never content, never a path), and not for matching on. +- **Not `Resolution.unavailable`.** The flag answers "may prune run?" and the token answers + "what does the caller learn?". They agree by construction — `unavailable` is `True` in + exactly the `store_unavailable` case — and both exist because `store_unavailable` must + stay distinct from `absent`: only a raising store suppresses pruning, since deleting + managed files after a failed lookup turns an outage into data loss. + +`get_skill_result` emits nothing of its own. The integrity log record and signal already +fired inside verification before `resolve_from_store` returned; recording anything here +would double-count one failure in a SIEM and in the product counter. + +There is no `get_skills_result` or `all_skills_result`. The batch accessors keep omitting +unresolved entries and keep logging the run-level WARN count, and a second accessor per +batch form would double the surface for a case nobody has asked for. + ### Security posture — do not relax any of this Store data is **untrusted input**; the transport is not part of the trust boundary. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 37b1570a..753e1e95 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -41,6 +41,7 @@ InMemorySkillStore, all_skills, get_skill, + get_skill_result, get_skills, skill_refs, ) @@ -85,6 +86,8 @@ ReconcileActionKind, ReconcileReport, Skill, + SkillOutcome, + SkillOutcomeReason, SkillReference, StreamChunkEvent, StreamDoneEvent, @@ -150,6 +153,7 @@ "ReconcileActionKind", "ReconcileReport", "Skill", + "SkillOutcome", "SkillReference", "StreamChunkEvent", "StreamDoneEvent", @@ -227,14 +231,16 @@ # skills "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the two closed-set unions a typed consumer needs to name + # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", + "SkillOutcomeReason", # skills — on-disk constants, identical across languages "SKILL_FILENAME", "MANIFEST_FILENAME", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index 9d9b3627..cb21986b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -37,7 +37,7 @@ resolve_from_store, verify_raw_skill, ) -from .types import AiConfigRep, Skill, SkillReference +from .types import AiConfigRep, Skill, SkillOutcome, SkillReference from .types_validation import ( is_valid_skill_key, is_valid_skill_version, @@ -247,6 +247,40 @@ async def get_skill(key: str, *, version: int | None = None) -> Skill | None: return resolve_from_store(require_store(), key, version).skill +async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome: + """ + Retrieves one verified skill, reporting *why* when there is none. + + Same retrieval, same verification, same telemetry as ``get_skill`` — the two + differ only in what they report. ``get_skill`` collapses "no such skill", + "the store raised", "that is not the version held", and "the content failed + integrity verification" to one ``None``; this returns a ``SkillOutcome`` + whose ``reason`` names which of them happened, so a caller can fail closed on + suspected tampering while tolerating a merely-absent skill: + + ```python + outcome = await get_skill_result("pdf-extraction") + if outcome.reason == "integrity_failure": + raise SystemExit(f"refusing to run: {outcome.detail}") + if outcome.skill is not None: + print(outcome.skill.content) + ``` + + ``detail`` is human-readable and safe to surface — it names the key and the + failure mode, never any skill content or filesystem path. Branch on + ``reason``, not on ``detail``. + + Emits nothing of its own: an integrity failure has already recorded its log + record and its signal inside verification, and recording a second here would + double-count one failure. Raises ``RuntimeError`` only when no skill store is + configured, exactly as ``get_skill`` does. + """ + resolved = resolve_from_store(require_store(), key, version) + return SkillOutcome( + skill=resolved.skill, reason=resolved.reason, detail=resolved.error + ) + + async def get_skills(refs: Sequence[SkillReference | str]) -> list[Skill]: """ Retrieves a batch of verified skills. diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 1e517e4e..74cb706b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -44,7 +44,7 @@ from dataclasses import dataclass from typing import Any, Literal, Protocol, get_args -from .types import Skill, SkillReference +from .types import Skill, SkillOutcomeReason, SkillReference from .types_validation import is_valid_skill_key, is_valid_skill_version logger = logging.getLogger(__name__) @@ -672,6 +672,26 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: class Resolution: """One key resolved against a store: the skill, or why there is none.""" + reason: SkillOutcomeReason + """ + Which of the five public outcomes this resolution is. + + Declared first and **without a default**, so every construction site has to + state it. A default would be the wrong shape twice over: a contributor + adding a sixth internal outcome would inherit whichever token happened to be + the default rather than deciding which public token it maps to, and if that + default were ``"ok"`` a failure would publish ``ok`` with no skill attached. + + Carried as a token rather than derived from ``error`` on the way out: + ``get_skill_result`` publishes this value, and pattern-matching prose to + recover a decision a caller fails closed on is exactly the fragility the + typed outcome exists to remove. A reviewer can read the mapping here. + + Distinct from ``unavailable`` on purpose — that flag answers one question + (may prune run?) and this token answers a different one (what does the + caller learn?) — but the two can only disagree by a bug: ``unavailable`` is + ``True`` in exactly the ``store_unavailable`` case. + """ skill: Skill | None = None error: str | None = None unavailable: bool = False @@ -702,26 +722,33 @@ def resolve_from_store( raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) except Exception as exc: logger.error("Skill store raised while retrieving '%s'", key, exc_info=True) - return Resolution(error=store_raised(exc), unavailable=True) + return Resolution( + reason="store_unavailable", + error=store_raised(exc), + unavailable=True, + ) if not isinstance(raw, dict): return Resolution( - error=f"skill '{key}' is not available from the configured skill store" + reason="absent", + error=f"skill '{key}' is not available from the configured skill store", ) skill = verify_raw_skill(raw) if skill is None: return Resolution( - error=f"skill '{key}' failed integrity verification and was withheld" + reason="integrity_failure", + error=f"skill '{key}' failed integrity verification and was withheld", ) if wanted_version is not None and skill.version != wanted_version: return Resolution( + reason="wrong_version", error=( f"skill '{key}' version {wanted_version} is not available " f"(the store holds version {skill.version})" - ) + ), ) - return Resolution(skill=skill) + return Resolution(reason="ok", skill=skill) def reference_target(item: SkillReference | str) -> tuple[str, int | None]: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 8a865dd8..fb163796 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -404,11 +404,17 @@ def _resolve_reference( """ store = _available_store(deadline, f"'{key}'") if isinstance(store, _RetrievalBlocked): - return Resolution(error=store.reason, unavailable=True) + return Resolution( + reason="store_unavailable", error=store.reason, unavailable=True + ) resolved = resolve_from_store(store, key, wanted_version) if resolved.unavailable and resolved.error is not None: - return Resolution(error=_unavailable(resolved.error), unavailable=True) + return Resolution( + reason="store_unavailable", + error=_unavailable(resolved.error), + unavailable=True, + ) return resolved diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 10c90d8c..65e4850a 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -452,6 +452,61 @@ class Skill: """Description from LaunchDarkly metadata; never parsed from the content.""" +SkillOutcomeReason = Literal[ + "absent", "integrity_failure", "ok", "store_unavailable", "wrong_version" +] +""" +The closed set of outcomes ``get_skill_result`` reports. + +Alphabetical, as ``reason_code`` is in the integrity log record, so the token +list reads identically in every LaunchDarkly AI SDK. Each token is a distinct +*decision* a caller can make, which is the point of the type: ``absent`` is a +skill the store does not hold, ``integrity_failure`` is content that was +delivered and did not verify, and a caller that wants to fail closed on +suspected tampering while tolerating a merely-absent skill needs the two to be +told apart. + +- ``ok`` — a verified skill was returned. +- ``absent`` — the store answered, and does not hold the key. +- ``integrity_failure`` — content was delivered and failed verification; it was + withheld. The one token worth failing closed on. +- ``store_unavailable`` — the store itself could not answer: it raised. + Deliberately distinct from ``absent``, because an outage is not a deletion. +- ``wrong_version`` — the store answered with a version other than the one + asked for, so the answer was withheld. +""" + + +@dataclass(frozen=True) +class SkillOutcome: + """ + Why one retrieval returned what it did — the reported form of ``get_skill``. + + ``get_skill`` collapses every failure to ``None``, which is the right shape + for a caller that only wants content and cannot act on the difference. This + is the shape for a caller that can: ``reason`` names which of the five + outcomes happened, so an integrity failure is distinguishable from a skill + that simply is not configured. The two accessors differ only in what they + report — the retrieval, the verification, and the telemetry are the same + code path, run once. + + Instances are immutable. + """ + + skill: Skill | None + """The verified skill, and only ever populated when ``reason == "ok"``.""" + reason: SkillOutcomeReason + """Which outcome happened. A closed set — see ``SkillOutcomeReason``.""" + detail: str | None + """ + Human-readable detail, set for every reason except ``ok``. + + Safe to log or surface to an operator: it carries the skill key and the + failure mode, and never any skill content or filesystem path. Intended for a + human, not for matching on — branch on ``reason``. + """ + + ReconcileActionKind = Literal[ "written", "updated", "skipped_current", "removed", "error" ] diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 0c656ede..6470f0c0 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -20,10 +20,12 @@ ReconcileAction, ReconcileReport, Skill, + SkillOutcome, SkillReference, all_skills, get_client, get_skill, + get_skill_result, get_skills, init_client, shutdown, @@ -143,6 +145,16 @@ def test_skill_is_immutable(self) -> None: with pytest.raises(dataclasses.FrozenInstanceError): skill.content = b"tampered" # type: ignore[misc] + def test_skill_outcome_is_immutable(self) -> None: + """A reported outcome is a value, like every other public skills type. + + Matters more here than for the others: a caller that fails closed on + ``reason`` must not be handed something a later layer can rewrite. + """ + outcome = SkillOutcome(skill=None, reason="integrity_failure", detail="nope") + with pytest.raises(dataclasses.FrozenInstanceError): + outcome.reason = "ok" # type: ignore[misc] + def test_skill_content_is_bytes(self) -> None: """Content is the verified verbatim bytes — opaque, never text.""" skill = _skill() @@ -385,6 +397,13 @@ def test_exported_action_union_admits_exactly_the_five_actions(self) -> None: "error", } assert set(typing.get_args(package.OnUnavailable)) == {"keep", "raise"} + assert set(typing.get_args(package.SkillOutcomeReason)) == { + "absent", + "integrity_failure", + "ok", + "store_unavailable", + "wrong_version", + } def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: """A name absent from ``__all__`` is not part of the public surface.""" @@ -393,11 +412,14 @@ def test_retrieval_surface_is_exported_from_the_package_root(self) -> None: expected = { "skill_refs", "get_skill", + "get_skill_result", "get_skills", "all_skills", "SkillStore", "InMemorySkillStore", "Skill", + "SkillOutcome", + "SkillOutcomeReason", "SkillReference", } assert expected <= set(package.__all__) @@ -743,6 +765,265 @@ async def test_multibyte_content_verifies( assert skill.content == content.encode("utf-8") +class _RaisingStore: + """A store whose reads raise — the "the transport is down" case. + + Declared with the full ``get_object`` signature on purpose. A double missing + the ``version`` parameter would also produce a raise here, but a + ``TypeError`` from the call itself rather than from the store, and the test + would then pass without the store ever having been consulted. + """ + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + raise RuntimeError("transport failure") + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + raise RuntimeError("transport failure") + + +class _WrongVersionAnsweringStore: + """A store that answers a pinned lookup with some other version.""" + + def __init__(self, make_raw_skill: Any, answered_version: int = 99) -> None: + self._make = make_raw_skill + self._answered_version = answered_version + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + answer: dict[str, Any] = self._make(key=key, version=self._answered_version) + return answer + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + +class TestGetSkillResult: + """ + The reported accessor — one token per outcome a retrieval can have. + + ``get_skill`` collapses four distinct failures to ``None``, which leaves a + caller unable to fail closed on suspected tampering while tolerating a skill + that is merely not configured. These tests pin that the five outcomes are + told apart, and that reporting them changed nothing about ``get_skill``. + """ + + def _tampered(self, make_raw_skill: Any, key: str = "a") -> dict[str, Any]: + raw: dict[str, Any] = make_raw_skill(key=key) + raw["contentHash"] = "0" * 64 + return raw + + async def test_ok_carries_the_skill_and_no_detail( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + store.put(make_raw_skill(key="pdf-extraction", version=2)) + + outcome = await get_skill_result("pdf-extraction") + + assert outcome.reason == "ok" + assert outcome.detail is None + assert outcome.skill is not None + assert outcome.skill.key == "pdf-extraction" + assert outcome.skill.version == 2 + assert outcome.skill.content == SKILL_BODY.encode("utf-8") + + async def test_absent_when_the_store_does_not_hold_the_key( + self, store: InMemorySkillStore + ) -> None: + outcome = await get_skill_result("nope") + + assert outcome.reason == "absent" + assert outcome.skill is None + assert outcome.detail + assert "'nope'" in outcome.detail + + async def test_integrity_failure_when_content_does_not_verify( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The one outcome a caller is expected to fail closed on.""" + store.put(self._tampered(make_raw_skill, key="a")) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # The quoted form, so the assertion is about the key and not about a + # letter that appears in half the words in the message. + assert "'a'" in outcome.detail + + async def test_wrong_version_when_the_store_answers_with_another( + self, make_raw_skill: Any + ) -> None: + skills_module._set_store(_WrongVersionAnsweringStore(make_raw_skill)) + + outcome = await get_skill_result("a", version=1) + + assert outcome.reason == "wrong_version" + assert outcome.skill is None + assert outcome.detail + # The detail is what makes this actionable rather than merely negative: + # it names both the version asked for and the version held. + assert "version 1" in outcome.detail + assert "version 99" in outcome.detail + + async def test_store_unavailable_when_the_store_raises( + self, caplog: pytest.LogCaptureFixture + ) -> None: + skills_module._set_store(_RaisingStore()) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "store_unavailable" + assert outcome.skill is None + assert outcome.detail + assert "RuntimeError" in outcome.detail + + async def test_store_unavailable_is_distinct_from_absent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """An outage is not a deletion, and the two must not read alike. + + This is the distinction ``write_skills`` already depends on to decide + whether pruning may run — only a raising store suppresses it — so + collapsing the two tokens here would put the public vocabulary at odds + with a policy the SDK already enforces internally. + """ + skills_module._set_store(_RaisingStore()) + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + raised = await get_skill_result("a") + + skills_module._set_store(InMemorySkillStore()) + empty = await get_skill_result("a") + + # Asserted as two named tokens rather than as an inequality: the type + # checker can already see that these two literals differ, so an + # inequality here would be dead weight. + assert raised.reason == "store_unavailable" + assert empty.reason == "absent" + + async def test_every_non_ok_outcome_carries_a_detail( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """A reason with no detail leaves an operator nothing to act on. + + Swept over all four failures in one test rather than asserted per case + only, so a fifth failure path added later without a message is caught by + a test whose name says what it is about. + """ + stores: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + outcomes: list[SkillOutcome] = [] + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for _expected, store_double in stores: + skills_module._set_store(store_double) + outcomes.append(await get_skill_result("a", version=1)) + + assert [o.reason for o in outcomes] == [expected for expected, _ in stores] + assert all(o.skill is None for o in outcomes) + assert all(o.detail for o in outcomes) + + async def test_detail_never_carries_the_skill_content( + self, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """``detail`` is safe to log, so the body must not travel in it. + + Same rule the integrity log record follows, asserted separately here + because this string reaches the caller through a different surface. + """ + secret = "---\nname: Secret\n---\nSSN 000-00-0000 and an API key.\n" + store.put(make_raw_skill(key="a", content=secret, contentHash="0" * 64)) + + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert outcome.detail is not None + assert secret not in outcome.detail + assert "SSN" not in outcome.detail + assert "API key" not in outcome.detail + + async def test_raises_the_same_way_as_get_skill_with_no_store(self) -> None: + """Identical failure mode, down to the message. + + The two accessors differ only in what they report about a retrieval; a + missing store is a configuration error in both, so a caller cannot need + to handle it twice. + """ + with pytest.raises(RuntimeError, match="skill store") as reported: + await get_skill_result("a") + with pytest.raises(RuntimeError, match="skill store") as collapsed: + await get_skill("a") + + assert str(reported.value) == str(collapsed.value) + + async def test_records_no_second_integrity_signal( + self, + store: InMemorySkillStore, + make_raw_skill: Any, + recording_emitter: Any, + caplog: pytest.LogCaptureFixture, + ) -> None: + """One failed retrieval is one failure, on both surfaces. + + Verification already recorded the log record and the signal before + ``resolve_from_store`` returned, so reporting the reason must add + nothing: a second record would double-count one event in a SIEM and + inflate the product counter. ``_integrity_records`` is the shared parser + used by the log-record tests further down this module. + """ + skills_module._set_emitter_for_testing(recording_emitter) + store.put(self._tampered(make_raw_skill, key="a")) + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + outcome = await get_skill_result("a") + + assert outcome.reason == "integrity_failure" + assert len(_integrity_records(caplog)) == 1 + assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + + async def test_get_skill_still_returns_none_for_every_failure( + self, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The no-behaviour-change guarantee. + + ``get_skill``'s contract — ``None`` for every failure, and it never + raises for one — is documented in its docstring and in the README, and + every existing caller treats ``None`` as "no skill". Adding a reported + accessor beside it must not move that line, so the four failures are + driven through both accessors in one test: the reason is distinguishable + *and* the collapsed form still collapses. + """ + cases: list[tuple[str, Any]] = [ + ("absent", InMemorySkillStore()), + ( + "integrity_failure", + InMemorySkillStore({"a": self._tampered(make_raw_skill)}), + ), + ("wrong_version", _WrongVersionAnsweringStore(make_raw_skill)), + ("store_unavailable", _RaisingStore()), + ] + + with caplog.at_level("ERROR", logger="launchdarkly_ai_server.skills_core"): + for expected_reason, store_double in cases: + skills_module._set_store(store_double) + reported = await get_skill_result("a", version=1) + assert reported.reason == expected_reason + # No pytest.raises wrapper: an escaping exception fails the test + # here, which is the "never raises" half of the contract. + assert await get_skill("a", version=1) is None + + class TestGetSkills: """Batch accessor.""" From 6a0e6aa50f6b528b10174d5834f013090817e962 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 4 Sep 2026 10:19:46 -0400 Subject: [PATCH 10/43] =?UTF-8?q?docs(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20close=20out=20the=20security=20review's=20SDK-side?= =?UTF-8?q?=20remainder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four items left open in the response to the Agent Skills security design review. Three are documentation, one is tests; no behavior changes, and the code halves of rows 9 and 2 are deliberately untouched. **Privilege separation** (row 9 docs half, and the agreed counter-proposal for row 26). The recommended deployment runs the reconcile as a different identity than the agent, which is the whole reason the ``0644``/``0755`` modes deny anything: the agent reads its instructions and cannot rewrite them, or the manifest. That is the mitigation for AZ-1, a prompt-injected agent editing its own skills. Write access to the manifest is the worse half — it is what tells the *next* reconcile which paths the SDK may delete — which is why ``_prune`` re-validates every entry from scratch rather than trusting it. The README documents the pattern and hands the operator the check to run, because the SDK cannot run it: it knows only its own identity, which trivially has write access, having just written there. So ``ReconcileReport`` grows no writability field — the review asked for one and we declined, since any check the SDK could make would answer a different question than the one asked and manufacture false confidence exactly where caution is wanted. ``agents.md`` records that reasoning so the field is not added later by someone reading its absence as an oversight. **Three hostile-manifest prune tests** (row 12 remainder): a well-formed manifest listing ``/etc/passwd``, ``../../../etc/passwd``, and a path under a parent that has since become a symlink. ``_prune`` already refuses all three, so these turn asserted into verified. Two things make them worth more than their line count. They are deliberately *well-formed* — the corrupt-manifest suite above them proves nothing here, because a corrupt manifest suppresses every destructive action wholesale, whereas these manifests give the implementation everything it needs to prune. And "deleted nothing" is asserted through an unlink spy rather than by checking that ``/etc/passwd`` still exists: the test process cannot delete that file anyway, so the obvious assertion would pass against an implementation with no path check at all. **One sentence on** ``"*"`` (row 16 remainder). It materializes the whole project library, so every skill's ``description`` enters the agent's context — including skills no AI Config references and skills belonging to other teams. **The Windows platform bound is now explicit** (row 2 residual), in ``safe_fs.py``, ``agents.md`` and the README. Reparse-point checks (``GetFileAttributesW`` / ``FILE_FLAG_OPEN_REPARSE_POINT``) are not implemented, by decision: Windows is not a supported or tested platform for this release, neither repository has a Windows CI runner so the checks would ship unverified, and the TypeScript SDK could not match them in any case because Node exposes no ``*at()`` family on *any* platform. Implementing them in Python alone would break cross-language parity and trade a documented bound for an unverified one. Two consequences are recorded rather than left to be rediscovered: on Windows, write permission on the managed root is the only boundary, which is what makes privilege separation the mitigation and not merely advice; and this retroactively lowers the priority of the row 25 reserved-device-name work, noted where that code lives. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 68 ++++++++- packages/client/agents.md | 25 ++++ .../src/launchdarkly_ai_server/safe_fs.py | 23 ++++ packages/client/tests/test_skills_fs.py | 130 ++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) diff --git a/packages/client/README.md b/packages/client/README.md index 045180b5..bc8df592 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -272,7 +272,12 @@ async def main(): asyncio.run(main()) ``` -Pass `"*"` instead of a reference list to materialize every skill the store holds. +Pass `"*"` instead of a reference list to materialize every skill the store holds — but know +what you are asking for. `"*"` materializes the **whole project library**, which puts every +skill's `description` into the agent's context, including skills no AI Config references and +skills belonging to other teams. `write_skills(skill_refs(...), root)` is the form used above +because it materializes only what the resolved variation actually asked for; reach for `"*"` +when you genuinely want the whole library on disk. **`skills` is now a validated field.** Config parsing fails closed on a `skills` value that is not a list of `{key, version}` objects (key matching `^[a-z0-9][a-z0-9-]*$`, version an @@ -406,6 +411,19 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. +**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX every destructive +step — the open, the rename, the unlink — runs relative to a directory descriptor opened +`O_RDONLY|O_DIRECTORY|O_NOFOLLOW` and held for the whole reconcile, so a directory swapped for +a symlink *after* its checks cannot redirect a write or a delete: the descriptor names the +inode that was checked, which closes the swap window rather than narrowing it. Windows has no +`*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check +taken immediately before each step. That floor is a check-then-use race rather than a closed +window: an attacker who already holds **write permission on the managed root** can still win +it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are +deliberately not implemented in this release, and Windows is not a tested platform for it — +neither SDK repository has a Windows CI runner. Treat write permission on the managed root as +the security boundary on every platform, and on Windows as the *only* one. + **One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is adopted — recorded in the manifest and reported `skipped_current` — rather than refused. @@ -477,6 +495,54 @@ to key its own map however the transport underneath does. > what was hashed. The SDK never parses or interprets them; if you want the frontmatter, > decode and parse the content on your side. +#### Privilege separation: the agent must not be able to rewrite its own skills + +**The recommended deployment runs `write_skills` as a different identity than the agent.** +Reconcile as one user, run the agent as another. Everything the reconcile puts on disk is +owner-write-only, and set explicitly rather than inherited from your umask: skill files and +the manifest at `0644` (via `fchmod` on the descriptor, so it cannot be redirected), the +per-skill `//` directories at `0755`, and the execute bit never set on anything. +Those modes are only a defense if the two identities actually differ — under a single identity +they describe a directory the agent can freely rewrite. + +**What to verify, as the identity that will run the agent.** The SDK cannot check this for you +(see below), so make it a deployment step: confirm the agent's identity has no write access to + +- the managed root itself, +- the per-skill directories `//` and the files `//SKILL.md`, +- the manifest at `/.launchdarkly-skills.json`. + +```bash +# Run as the agent's user. Every line should print DENIED. +root=.claude/skills +for target in "$root" "$root/.launchdarkly-skills.json" "$root"/*/ "$root"/*/SKILL.md; do + [ -e "$target" ] || continue + if [ -w "$target" ]; then echo "WRITABLE — fix this: $target"; else echo "DENIED: $target"; fi +done +``` + +Note that the managed root's own mode is **yours, not the SDK's**: `write_skills` creates only +that one leaf directory and does so with your umask, precisely because the root is a path you +chose. Own it — `chown reconcile-user:agent-group` and `chmod 0755` on the root is the shape +that makes the rest of the tree's modes mean something. + +**Why this is the mitigation that matters.** A `SKILL.md` is agent *instructions*. An agent +that can write its own skills directory can rewrite its own instructions, and an agent +processing untrusted input is exactly the thing that might be induced to do so. Write access +to the manifest is worse than write access to a skill, because the manifest is what tells the +*next* reconcile which paths the SDK owns and may delete: an agent that can edit it can keep a +skill LaunchDarkly has revoked, or aim the SDK's own delete path at something it should not +touch. `write_skills` re-validates every manifest entry from scratch for exactly that reason — +it treats that file as untrusted input, never as authorization — but an agent that cannot edit +it at all is the stronger position, and only your deployment can provide that. + +**The SDK deliberately does not report whether the root is writable.** There is no such field +on `ReconcileReport`, and its absence is a decision rather than an oversight. The SDK knows +only its own identity, which trivially has write access — it just wrote there. It cannot know +which identity will later run the agent, so any check it could make would answer a different +question than the one that matters, and would read as reassurance exactly where caution is +wanted. You know both identities; the SDK knows one. + --- ### Utility Helpers diff --git a/packages/client/agents.md b/packages/client/agents.md index a7b4ce1b..2558f9c3 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -510,6 +510,31 @@ resolving to the skill directory's `(st_dev, st_ino)`) instead of by comparing p A spy must `fstat` the descriptor **inside** the intercepted call — the implementation closes it as soon as the write returns. +**The platform bound is POSIX-only, and that is a decision — do not quietly "fix" it.** +Windows reparse-point checks (`GetFileAttributesW`, `FILE_FLAG_OPEN_REPARSE_POINT`) are not +implemented because Windows is not a supported or tested platform for this release: there is +no Windows CI runner in either repository, so the checks would ship unverified, and the +TypeScript SDK could not match them at all — Node exposes no `*at()` family on *any* +platform, so its racy floor is universal rather than Windows-only. Implementing them in +Python alone would break cross-language parity and trade a documented bound for an unverified +one. Two follow-on facts: on Windows write permission on the managed root is the only +boundary, which is why the privilege-separated deployment is documented as the mitigation +rather than as advice; and this bound retroactively lowers the priority of the reserved-device-name +work above — keep that code, but do not read it as evidence that Windows is hardened. If +Windows becomes a supported platform, revisit both together, and add the CI runner first. + +**Privilege separation is the deployment-side half of this, and `ReconcileReport` must not +grow a writability field.** The recommended deployment runs the reconcile as a different +identity than the agent, so the `0644`/`0755` modes above actually deny something: the agent +reads its instructions and cannot rewrite them or the manifest. That is the mitigation for a +prompt-injected agent editing its own skills. The security review asked for the report to +surface whether the managed root is writable; we declined, and the reasoning is load-bearing +rather than a preference. The SDK knows only its *own* identity, which trivially has write +access — it just wrote there — and cannot know which identity will later run the agent. Any +check it could perform would answer a different question than the one asked and would create +false confidence exactly where caution is wanted. The operator's verification steps live in +the README instead. Do not add the field. + ### Deferred: bounded retries `timeout` is implemented — a monotonic deadline, checked before each retrieval, before diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 73eef878..43c21e58 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -11,6 +11,29 @@ re-resolving a name — which is what closes the swap window rather than merely narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the identical sequence runs against full paths, the per-component ``lstat`` floor. + +**Platform bound — this guarantee is POSIX-only, deliberately.** On POSIX the +descriptor walk closes the swap window. On Windows it does not exist: there is no +``*at()`` family, so the ``lstat`` floor is all that runs, and a floor is a +check-then-use race rather than a closed window. The remedy would be +reparse-point checks (``GetFileAttributesW``, or opening with +``FILE_FLAG_OPEN_REPARSE_POINT``) and it is **not implemented, by decision rather +than by oversight**: Windows is not a supported or tested platform for this +release, and neither SDK repository has a Windows CI runner, so the checks would +ship untested — and the TypeScript SDK could not match them in any case, because +Node exposes no ``*at()`` family on *any* platform. Shipping them in Python alone +would break the cross-language parity the two SDKs are held to and would trade a +documented bound for an unverified one. + +Two consequences worth stating plainly rather than discovering later. First, on +Windows write permission on the managed root is the *only* boundary, so the +privilege-separated deployment the README documents is not advice there but the +mitigation. Second, this bound retroactively lowers the priority of the Windows +reserved-device-name work in ``skills_fs.py`` (``_WINDOWS_RESERVED_NAMES``): that +code stays, because it is cheap and it keeps a managed root written on Linux +usable when read from Windows, but it should not be read as evidence that Windows +is a hardened target. It is not. Revisit both together if Windows becomes +supported. """ from __future__ import annotations diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 223a3262..6d5ab2db 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1394,6 +1394,136 @@ async def test_corrupt_manifest_file_is_not_destroyed(self, root: Path) -> None: assert _manifest_path(root).read_text(encoding="utf-8") == "{not json at all" +# The three literal cases the security review names for the prune path. +# +# The distinction from ``TestCorruptManifest`` above is the whole point: a +# corrupt manifest suppresses every destructive action wholesale, so those tests +# say nothing about these. Each manifest here is *well-formed* — parseable, +# a ``manifestVersion`` this release understands, a real ``entries`` map, and an +# entry whose ``key`` is a perfectly valid skill key that is genuinely absent +# from the requested set. The implementation has every input it needs to prune +# and must refuse anyway, because the recorded *path* is not one this SDK could +# have written. +HOSTILE_RECORDED_PATHS: list[str] = [ + # Absolute: the classic. A recorded path read back and unlinked as-is is a + # delete of an attacker-chosen file with the reconcile's privileges. + "/etc/passwd", + # Traversing: the same attack for an implementation that rejects a leading + # slash and then joins the rest onto the root. + "../../../etc/passwd", +] + + +class _UnlinkSpy: + """Records every ``os.unlink`` while delegating to the real one. + + Asserting only that ``/etc/passwd`` still exists proves nothing: the test + process cannot delete it anyway, so that assertion passes against an + implementation with no path check at all — permissions would be doing the + work. What has teeth is that the removal is never *attempted*: the refusal + happens above the syscall, on a path the SDK recomputes rather than trusts. + """ + + def __init__(self) -> None: + self.targets: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _UnlinkSpy: + real = os.unlink + + def spy(path: Any, *args: Any, **kwargs: Any) -> None: + self.targets.append(os.fsdecode(path)) + real(path, *args, **kwargs) + + # ``safe_fs_module.os`` *is* the ``os`` module, so this covers both the + # descriptor-relative ``os.unlink(name, dir_fd=...)`` and the + # ``Path.unlink`` used on the no-``*at()`` floor. + monkeypatch.setattr(safe_fs_module.os, "unlink", spy) + return self + + +class TestHostileManifestPrune: + """A well-formed manifest naming a path this SDK could not have written. + + The manifest is untrusted input. It is a plain file on the customer's disk + that anything with write access to the managed root can edit, and ``prune`` + is the one code path in the SDK that deletes. So a recorded path never + authorizes its own removal: it must match ``/SKILL.md`` for a + re-validated key, and the target is recomputed from the *current* managed + root instead of being read back out of the entry. + """ + + @pytest.mark.parametrize("recorded", HOSTILE_RECORDED_PATHS) + async def test_recorded_path_outside_the_root_is_refused( + self, root: Path, recorded: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + spy = _UnlinkSpy().install(monkeypatch) + _write_manifest( + root, + { + "manifestVersion": 1, + "entries": {recorded: _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + # The refusal is about ownership of the path, not about the file's state. + assert "could own" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + # Nothing was even attempted, let alone completed. + assert spy.targets == [] + assert Path("/etc/passwd").exists() + # Left in place rather than tidied away: dropping the entry would let a + # single hostile edit erase the SDK's own record of what it manages. + assert recorded in _read_manifest(root)["entries"] + + async def test_entry_under_a_since_symlinked_parent_is_refused( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The recorded path is the SDK's own, and is still not enough. + + Here the entry is exactly what a legitimate reconcile writes — + ``a/SKILL.md`` under key ``a`` — so the shape check that catches the two + cases above passes. What changed is the disk underneath it: ``/a`` + is now a symlink to somewhere else. This is the case a validate-then-act + implementation fails, because the manifest and the entry are both + entirely legitimate; only the current state of the parent is not. + """ + root = tmp_path / "skills" + root.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + victim = elsewhere / "SKILL.md" + victim.write_text("victim content\n", encoding="utf-8") + + # Managed legitimately first, so the manifest entry is one this SDK + # really did write... + managed = _place_managed(root, "a", SKILL_BODY) + # ...then the parent directory is swapped for a link out of the root. + managed.unlink() + (root / "a").rmdir() + (root / "a").symlink_to(elsewhere, target_is_directory=True) + + spy = _UnlinkSpy().install(monkeypatch) + report = await write_skills([], root) + + assert report.ok is False + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert action.error is not None + assert "symlink" in action.error + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The file the symlink pointed at is untouched, and so is the link. + assert victim.read_text(encoding="utf-8") == "victim content\n" + assert (root / "a").is_symlink() + assert "a/SKILL.md" in _read_manifest(root)["entries"] + + class TestWriteSkillsTelemetry: """Materialized / revoked signals from write_skills.""" From 3bdbfabe41e029687a6b10a10fdd8f6d6a257a3e Mon Sep 17 00:00:00 2001 From: Patrick Kaeding Date: Fri, 4 Sep 2026 12:12:09 -0400 Subject: [PATCH 11/43] =?UTF-8?q?test(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20root-swap=20races=20the=20descriptor=20walk?= =?UTF-8?q?=20does=20not=20close?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three expected-failure tests (TestRootSwapRaces) for SEC-8985 row 2. The SDR response says every destructive operation runs relative to a descriptor held for the duration of the reconcile. The code pins / per operation and never holds the root: _resolve_root validates it once and returns a path, and each write and prune re-opens / by path with O_NOFOLLOW, which guards only the final component. A root swapped for a symlink after validation redirects the open, and every descriptor-relative step behind it, into the attacker's directory. Precondition is write permission on the root's parent, which the README checklist does not mention. The tests state the contract (nothing lands outside the root, no outside file is overwritten, no outside file is removed) and are marked xfail(strict=True, raises=AssertionError) so the suite stays green, the gap is recorded next to the other race tests, and the fix cannot land without removing the marker. Run with --runxfail to see the three escapes. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit c22688b9238a1c21de2f72acf0c5b67b79e82fee) --- packages/client/tests/test_skills_fs.py | 175 ++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 6d5ab2db..9c0d1012 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -198,6 +198,56 @@ def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapDirectoryDuring: return self +class _SwapRootDuring: + """Fires the *root*-swap race at the exact instant of an operation. + + ``_SwapDirectoryDuring`` one level up: renames the managed root itself aside + and leaves a symlink to *outside* in its place, then lets the intercepted + call proceed. ``O_NOFOLLOW`` guards only the final path component, so an + ``os.mkdir`` or ``os.open`` of ``/`` issued after the swap is + resolved by the kernel *through* the link: the descriptor that comes back is + pinned to ``/``, and every descriptor-relative step that + follows is relative to the wrong directory. + + The attacker capability is write permission on the root's *parent*, not on + the root. The README's privilege-separation checklist denies the agent + identity the root, the skill directories, the files and the manifest, and + says nothing about the parent; in the documented layout + (``/.claude/skills``) that parent is ``.claude``, which the agent + identity typically owns. + + Fires once, on the first call whose path argument is ``/``; every + other call passes straight through. + """ + + def __init__(self, attribute: str, root: Path, key: str, outside: Path) -> None: + self.attribute = attribute + self.root = Path(os.path.realpath(root)) + self.trigger = self.root / key + self.moved_to = self.root.parent / f"{self.root.name}.real" + self.outside = outside + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + # ``os.mkdir(path, mode)`` and ``os.open(path, flags, ...)`` both take the + # path first; descriptor-relative calls pass a bare name, which never + # compares equal to the absolute trigger. + if ( + not self.swapped + and isinstance(first, (str, os.PathLike)) + and Path(os.fspath(first)) == self.trigger + ): + os.rename(self.root, self.moved_to) + os.symlink(self.outside, self.root, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootDuring: + monkeypatch.setattr(safe_fs_module.os, self.attribute, self) + return self + + _needs_dir_fd = pytest.mark.skipif( not safe_fs_module.SUPPORTS_DIR_FD, reason="no *at() family on this platform; the per-component lstat floor applies", @@ -1104,6 +1154,131 @@ async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( assert [a.action for a in report.actions if a.key == "a"] == ["removed"] +_root_not_held = pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "SEC-8985 row 2: the managed root is validated once but never held open, " + "so a root swapped for a symlink before / is opened redirects " + "the operation out of the root. Remove this marker once write_skills pins " + "the root and walks from it (dir_fd=root_fd)." + ), +) +""" +Strict, so the day the fix lands these tests fail as XPASS and the marker has +to go — they cannot silently stay "expected to fail" past the fix. Narrowed to +``AssertionError`` so a harness breakage (an unexpected ``OSError``, a race +that never fires) is reported as a real failure rather than absorbed as the +known gap. +""" + + +@pytest.mark.skipif( + not hasattr(os, "symlink"), reason="platform has no symlink support" +) +class TestRootSwapRaces: + """The swap one level up: the managed *root*, not ``/``. + + ``_resolve_root`` validates the root once and returns a path; nothing holds + it open. Each write and each prune then opens ``/`` *by path* + with ``O_NOFOLLOW | O_DIRECTORY`` and pins that. ``O_NOFOLLOW`` guards only + the final component, so the root and every ancestor are re-resolved on + every such open, and a root swapped for a symlink after validation + redirects the open — and with it every descriptor-relative step behind it + — into the attacker's directory. ``atomic_write_in`` does refuse the + manifest rewrite afterwards, but by then the skill file is already outside + the root. + + ``TestSymlinkAttacks`` proves the ``/`` swap is closed. These + three state the same contract for the root — nothing lands outside it, no + outside file is overwritten, no outside file is removed — and are marked + expected-to-fail because the code does not yet meet it. The fix is to pin + the root once at the top of ``write_skills`` and walk from it: + ``os.mkdir(key, dir_fd=root_fd)``, ``os.open(key, ..., dir_fd=root_fd)``, + and ``atomic_write(..., dir_fd=root_fd)`` for the manifest. + """ + + @_needs_dir_fd + @_root_not_held + async def test_root_swapped_at_the_skill_directory_create_cannot_redirect_the_write( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """First reconcile against a fresh root: the skill directory does not + exist yet, so ``open_or_create_directory`` calls ``os.mkdir(/a)``. + The swap fires there; the ``mkdir`` and the ``O_NOFOLLOW`` open that + follows both resolve through the link, and ``SKILL.md`` is written into + ``/a/``.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + race = _SwapRootDuring("mkdir", root, "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + if not race.swapped: + pytest.fail("the race never fired; the test proves nothing") + assert list(outside.iterdir()) == [] + # Either the skill landed in the real root or the run says it did not. + assert report.ok is False or ( + (race.moved_to / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + ) + + @_needs_dir_fd + @_root_not_held + async def test_root_swapped_at_the_skill_directory_open_cannot_clobber_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Update of an already-managed skill: the swap fires at the first + ``O_NOFOLLOW`` open of ``/a`` (the orphan-temp sweep's), and the + write re-opens by path and inherits it. ``/a/SKILL.md`` — a + file the manifest never recorded — is replaced with LaunchDarkly-served + content.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootDuring("open", root, "a", outside).install(monkeypatch) + + report = await write_skills([_skill("a", 2, "served update\n")], root) + + if not race.swapped: + pytest.fail("the race never fired; the test proves nothing") + assert victim.read_text(encoding="utf-8") == "precious\n" + assert report.ok is False or ( + (race.moved_to / "a" / "SKILL.md").read_text(encoding="utf-8") + == "served update\n" + ) + + @_needs_dir_fd + @_root_not_held + async def test_root_swapped_at_the_prune_cannot_redirect_the_unlink( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The destructive side. A prune of a formerly-managed key opens + ``/a`` by path, unlinks ``SKILL.md`` relative to that descriptor, + then ``rmdir``s the directory — all three resolve through the swapped + root, so an outside file and its directory are removed.""" + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootDuring("open", root, "a", outside).install(monkeypatch) + + report = await write_skills([], root) + + if not race.swapped: + pytest.fail("the race never fired; the test proves nothing") + assert victim.exists() and victim.read_text(encoding="utf-8") == "precious\n" + assert report.ok is False or not (race.moved_to / "a" / "SKILL.md").exists() + + class TestWithoutDirFd: """The full-path fallback for platforms with no ``*at()`` family. From 559c98551b0554ae15ab765eeeb7c0d06c14f99c Mon Sep 17 00:00:00 2001 From: Patrick Kaeding Date: Fri, 4 Sep 2026 12:41:25 -0400 Subject: [PATCH 12/43] test(client): let the root-swap races fail outright The expected-failure marker made the suite green while the contract was violated. A red run is the demonstration: the tests assert the contract, the code does not meet it, and they go green when the root is pinned for the reconcile, with nothing to remove. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 37ddb6aa7e4241bf5dfb8b6ff30e27ed0eb588be) --- packages/client/tests/test_skills_fs.py | 37 ++++--------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 9c0d1012..812eff72 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1154,25 +1154,6 @@ async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( assert [a.action for a in report.actions if a.key == "a"] == ["removed"] -_root_not_held = pytest.mark.xfail( - strict=True, - raises=AssertionError, - reason=( - "SEC-8985 row 2: the managed root is validated once but never held open, " - "so a root swapped for a symlink before / is opened redirects " - "the operation out of the root. Remove this marker once write_skills pins " - "the root and walks from it (dir_fd=root_fd)." - ), -) -""" -Strict, so the day the fix lands these tests fail as XPASS and the marker has -to go — they cannot silently stay "expected to fail" past the fix. Narrowed to -``AssertionError`` so a harness breakage (an unexpected ``OSError``, a race -that never fires) is reported as a real failure rather than absorbed as the -known gap. -""" - - @pytest.mark.skipif( not hasattr(os, "symlink"), reason="platform has no symlink support" ) @@ -1191,15 +1172,14 @@ class TestRootSwapRaces: ``TestSymlinkAttacks`` proves the ``/`` swap is closed. These three state the same contract for the root — nothing lands outside it, no - outside file is overwritten, no outside file is removed — and are marked - expected-to-fail because the code does not yet meet it. The fix is to pin - the root once at the top of ``write_skills`` and walk from it: + outside file is overwritten, no outside file is removed. The code does not + yet meet it, so they fail, and the red run is the demonstration. The fix is + to pin the root once at the top of ``write_skills`` and walk from it: ``os.mkdir(key, dir_fd=root_fd)``, ``os.open(key, ..., dir_fd=root_fd)``, and ``atomic_write(..., dir_fd=root_fd)`` for the manifest. """ @_needs_dir_fd - @_root_not_held async def test_root_swapped_at_the_skill_directory_create_cannot_redirect_the_write( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1216,8 +1196,7 @@ async def test_root_swapped_at_the_skill_directory_create_cannot_redirect_the_wr report = await write_skills([_skill("a")], root) - if not race.swapped: - pytest.fail("the race never fired; the test proves nothing") + assert race.swapped is True, "the race never fired; the test proves nothing" assert list(outside.iterdir()) == [] # Either the skill landed in the real root or the run says it did not. assert report.ok is False or ( @@ -1225,7 +1204,6 @@ async def test_root_swapped_at_the_skill_directory_create_cannot_redirect_the_wr ) @_needs_dir_fd - @_root_not_held async def test_root_swapped_at_the_skill_directory_open_cannot_clobber_an_outside_file( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1245,8 +1223,7 @@ async def test_root_swapped_at_the_skill_directory_open_cannot_clobber_an_outsid report = await write_skills([_skill("a", 2, "served update\n")], root) - if not race.swapped: - pytest.fail("the race never fired; the test proves nothing") + assert race.swapped is True, "the race never fired; the test proves nothing" assert victim.read_text(encoding="utf-8") == "precious\n" assert report.ok is False or ( (race.moved_to / "a" / "SKILL.md").read_text(encoding="utf-8") @@ -1254,7 +1231,6 @@ async def test_root_swapped_at_the_skill_directory_open_cannot_clobber_an_outsid ) @_needs_dir_fd - @_root_not_held async def test_root_swapped_at_the_prune_cannot_redirect_the_unlink( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1273,8 +1249,7 @@ async def test_root_swapped_at_the_prune_cannot_redirect_the_unlink( report = await write_skills([], root) - if not race.swapped: - pytest.fail("the race never fired; the test proves nothing") + assert race.swapped is True, "the race never fired; the test proves nothing" assert victim.exists() and victim.read_text(encoding="utf-8") == "precious\n" assert report.ok is False or not (race.moved_to / "a" / "SKILL.md").exists() From dd5db3718ccf571bf42cde2bbc092159a8066b95 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 4 Sep 2026 15:23:01 -0400 Subject: [PATCH 13/43] fix(client): pin the skills root for the whole reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEC-8985 row 2. The SDR response says every destructive operation runs relative to a descriptor held for the duration of the reconcile. It did not. `_resolve_root` validated the root and returned a plain `Path`; nothing held it open. Each write and each prune then opened `/` by path with `O_NOFOLLOW|O_DIRECTORY` and pinned that — and `O_NOFOLLOW` guards only the final component, so the root and every ancestor were re-resolved on every such open. A root renamed aside and replaced with a symlink after validation redirected the open, and with it every descriptor-relative step behind it, into the attacker's directory; on the create path `os.mkdir(/)` followed the link as well, and `mkdir` follows a symlink at its parent. The manifest write was the one operation that pinned the root, and it ran last, by which time the skill files were already outside it. The attacker precondition is write access to the root's *parent* — `.claude` for a root of `.claude/skills` — which the README checklist did not mention. `write_skills` now opens the root once, immediately after `_resolve_root`, with `O_RDONLY|O_DIRECTORY|O_NOFOLLOW`, confirms `S_ISDIR` on the descriptor, and holds it until the call returns. The descriptor is threaded through `_write_all`, `_prune`, `_rewrite_manifest`, the orphan sweep and the per-skill helpers, and every destructive step names a bare component against it: `os.mkdir(key, dir_fd=root_fd)`, `os.open(key, ..., dir_fd=root_fd)` for the skill directory, `os.rmdir(key, dir_fd=root_fd)`, and `atomic_write(..., dir_fd=root_fd)` for the manifest. A root swapped in the one interval left — after validation, before the open — fails `O_NOFOLLOW` and is reported as a run-level error with nothing touched, rather than as the `ValueError` an unusable root raises. `safe_fs`'s three openers take a `dir_fd` for the *parent* rather than growing a parallel API, and `SUPPORTS_DIR_FD` now probes `os.mkdir` and `os.rmdir` alongside the four it already named. Where the `*at()` family is absent the per-component `lstat` floor runs exactly as before: the root open returns `None` there, and every call site keeps its full-path branch. `_unsafe_path_reason` stays and still runs, but it is documented as defense in depth rather than the boundary — every check in it inspects a path, so each is a check-then-use against anything that can rename a component of that path. Security's three tests fired only when the intercepted `os.mkdir`/`os.open` was handed the absolute `/`, which the fix stops passing — they would have gone green while asserting nothing. The trigger now matches the bare key as well, so the swap fires in both worlds and the tests fail before the fix and pass after it. Two tests added: the root swapped before the pin is refused at the run level, and an audit that across a full reconcile no destructive call names an absolute path. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/safe_fs.py | 88 ++++++-- .../src/launchdarkly_ai_server/skills_fs.py | 209 +++++++++++++----- packages/client/tests/test_skills_fs.py | 177 ++++++++++++--- 3 files changed, 377 insertions(+), 97 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 43c21e58..34bafdec 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -53,8 +53,9 @@ and never executable.""" SUPPORTS_DIR_FD = os.supports_dir_fd.issuperset( - # renameat, openat, unlinkat, fstatat — the four this module needs. - {os.rename, os.open, os.unlink, os.stat} + # renameat, openat, unlinkat, fstatat, mkdirat, and unlinkat's AT_REMOVEDIR + # form — the six this module and its caller need. + {os.rename, os.open, os.unlink, os.stat, os.mkdir, os.rmdir} ) """ Whether the ``*at()`` syscall family is available, so every operation under the @@ -67,6 +68,11 @@ root. POSIX has these calls; Windows does not, and there the per-component ``lstat`` floor the spec permits applies instead. +``os.mkdir`` and ``os.rmdir`` are in the set because the managed root is now +pinned for a whole reconcile and the per-skill directory is created and removed +relative to that descriptor. Both are registered by CPython under the names +called here, so unlike the two below they need no indirection. + The probe deliberately names ``os.rename`` and ``os.stat`` rather than the ``os.replace`` and ``os.lstat`` this module actually calls. ``os.supports_dir_fd`` is populated per underlying syscall, and CPython registers ``renameat`` under @@ -79,10 +85,33 @@ """ -def open_directory_nofollow(directory: Path) -> int | None: +def _at(directory: Path, dir_fd: int | None) -> str | Path: + """ + What to name *directory* by, given a descriptor for its parent. + + With a *dir_fd* the call must pass the bare final component, so the kernel + resolves it inside the pinned parent and no ancestor is re-resolved from its + path; without one the full path is the only thing there is to pass. Spelled + once because every operation in this module that accepts a parent + descriptor has to make the same choice, and one call site left on the full + path would silently re-open the window the descriptor closes. + """ + return directory.name if dir_fd is not None else directory + + +def open_directory_nofollow( + directory: Path, *, dir_fd: int | None = None +) -> int | None: """ Opens *directory* without following a final symlink, and pins it. + *dir_fd* is a descriptor for the *parent*, and passing one is what extends + the guarantee past the final component: ``O_NOFOLLOW`` refuses a link at + *directory* itself, but every ancestor above it is re-resolved from its path + on each open, so a parent swapped for a symlink after it was checked + redirects the open. Given a parent descriptor the bare name is resolved + inside the inode that was checked instead, and there is nothing left to swap. + Everything the caller does afterwards goes through the returned descriptor instead of the path, which is what turns the "narrow window" into no window at all: the descriptor names the inode that was checked, so swapping @@ -113,7 +142,7 @@ def open_directory_nofollow(directory: Path) -> int | None: flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) try: - fd = os.open(directory, flags) + fd = os.open(_at(directory, dir_fd), flags, dir_fd=dir_fd) except OSError as exc: raise ValueError( f"the directory could not be opened without following links: {exc}" @@ -129,7 +158,9 @@ def open_directory_nofollow(directory: Path) -> int | None: return fd -def open_or_create_directory(directory: Path) -> int | None: +def open_or_create_directory( + directory: Path, *, dir_fd: int | None = None +) -> int | None: """ Creates *directory* if absent and returns a descriptor pinned to it. @@ -137,20 +168,40 @@ def open_or_create_directory(directory: Path) -> int | None: "already there", which would re-open the very hole the caller's ``lstat`` check just closed. ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` path does not: a link reports as a link, and is refused. + + *dir_fd* is a descriptor for the parent, as in ``open_directory_nofollow``, + and the ``mkdir`` needs it every bit as much as the open does: ``mkdir`` + follows a symlink at the parent, so a create issued against the full path is + how a directory gets made — and then written into — outside the root. """ + # As in ``atomic_write``: a parent descriptor is only usable where the + # ``*at()`` family is. ``open_directory_nofollow`` returns ``None`` on the + # platforms without it, so no caller here can hold one — stated rather than + # left to that invariant, because the ``mkdir`` below would otherwise raise + # instead of taking the full-path floor the openers fall back to. + if not SUPPORTS_DIR_FD: + dir_fd = None try: - os.mkdir(directory, 0o755) + os.mkdir(_at(directory, dir_fd), 0o755, dir_fd=dir_fd) except FileExistsError: - mode = os.lstat(directory).st_mode + # os.stat(follow_symlinks=False) on the descriptor path, os.lstat off it: + # identical results, and the former is the spelling os.supports_dir_fd + # advertises. + if dir_fd is not None: + mode = os.stat(directory.name, dir_fd=dir_fd, follow_symlinks=False).st_mode + else: + mode = os.lstat(directory).st_mode if stat.S_ISLNK(mode): raise ValueError("the directory is a symlink") from None if not stat.S_ISDIR(mode): raise ValueError("the path is not a directory") from None - return open_directory_nofollow(directory) + return open_directory_nofollow(directory, dir_fd=dir_fd) @contextmanager -def pinned_directory(directory: Path, *, create: bool = False) -> Iterator[int | None]: +def pinned_directory( + directory: Path, *, create: bool = False, dir_fd: int | None = None +) -> Iterator[int | None]: """ Holds *directory* pinned for the duration of the block, then releases it. @@ -158,11 +209,15 @@ def pinned_directory(directory: Path, *, create: bool = False) -> Iterator[int | ``lstat`` floor — so the caller states the platform split once, as ``if dir_fd is not None``, and cannot forget the ``os.close``. Raises ``ValueError`` for a directory that will not pin, exactly as they do. + + *dir_fd* is a descriptor for the parent and is passed straight through. Note + which descriptor is which: the one passed *in* pins the parent, and the one + yielded pins *directory* itself. """ dir_fd = ( - open_or_create_directory(directory) + open_or_create_directory(directory, dir_fd=dir_fd) if create - else open_directory_nofollow(directory) + else open_directory_nofollow(directory, dir_fd=dir_fd) ) try: yield dir_fd @@ -353,10 +408,13 @@ def atomic_write_in(directory: Path, name: str, data: bytes) -> None: """ ``atomic_write`` against a directory this module does not already hold open. - Used for the skills manifest, whose directory is the managed root. The - descriptor is taken with ``O_NOFOLLOW``, so a root swapped for a symlink after - ``_resolve_root`` validated it fails the write instead of redirecting it — - the caller turns that into a run-level ``error`` action. + The descriptor is taken with ``O_NOFOLLOW``, so a directory swapped for a + symlink between the caller's checks and the write fails it rather than + redirecting it. That still leaves the directory's *ancestors* re-resolved on + the open, so this is the right primitive only where the caller holds nothing + better. ``skills_fs`` no longer does — it pins the managed root for the whole + reconcile and writes the manifest with ``atomic_write(..., dir_fd=root_fd)`` + — so prefer passing a descriptor over reaching for this. """ with pinned_directory(directory) as dir_fd: atomic_write(directory, name, data, dir_fd=dir_fd) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index fb163796..2666bee1 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -8,6 +8,13 @@ filesystem. The dependency runs one way only, and the descriptor-pinned primitives every destructive step goes through live in ``safe_fs.py``. +The managed root is pinned to a descriptor once per reconcile and every +destructive operation runs relative to it, so no ancestor of the root is ever +re-resolved from its path mid-run. The path checks are still there and still +run, but they are defense in depth rather than the boundary: a check on a path +is only as good as the last resolution of that path after it, and the descriptor +is what makes there be no later resolution. + The reconcile is manifest-driven and fails closed: destructive operations only ever touch paths ``/.launchdarkly-skills.json`` records under a matching key, a corrupt manifest suppresses every destructive action, and an incomplete @@ -32,8 +39,8 @@ from .safe_fs import ( SymlinkRefused, atomic_write, - atomic_write_in, is_temp_name, + open_directory_nofollow, pinned_directory, unlink_file, ) @@ -160,6 +167,12 @@ async def write_skills( Returns a ``ReconcileReport`` in which every outcome is visible; raises ``ValueError`` for a caller error such as an unusable root. + The root is opened once, up front, and that descriptor is held until the + call returns: every directory created, file written and file removed under + it is resolved relative to the descriptor rather than from the root's path, + so a root replaced after validation is refused rather than followed. This + guarantee is POSIX-only, for the reasons ``safe_fs`` states at length. + **This call performs synchronous filesystem I/O and does not yield.** It is ``async`` for signature parity with the other accessors and with the TypeScript SDK, not because it awaits anything: every read, write, ``fsync`` @@ -186,38 +199,73 @@ async def write_skills( deadline = time.monotonic() + timeout root_path = _resolve_root(root) - manifest, manifest_error = _load_manifest(root_path) - entries: dict[str, Any] = manifest.get("entries", {}) - actions: list[ReconcileAction] = [] - if manifest_error is not None: - # Run-level failure: there is no single skill key to hang it off. - actions.append(_run_error(manifest_error)) - - requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) - - written, write_timed_out = _write_all(root_path, requests, entries, deadline) - actions.extend(written) - incomplete = incomplete or write_timed_out - - # Pruning is destructive, so it needs a trustworthy picture of both sides: a - # corrupt manifest means we do not know what we own, and an incomplete run — - # a retrieval that failed, or a deadline that expired mid-write — means we do - # not know what is still current. Either way, deleting would be a guess. - if prune and manifest_error is None and not incomplete: - actions.extend( - _prune( - root_path, - entries, - {request.key for request in requests}, - deadline, - ) + # The root is pinned once, here, and held for the whole reconcile. Every + # destructive step below runs relative to this descriptor, which is what + # makes _resolve_root's validation mean something afterwards: O_NOFOLLOW on + # a per-operation open of / guards only that final component, so + # the root and its ancestors were re-resolved on every such open and a root + # renamed aside and replaced with a symlink after validation redirected the + # open — and every descriptor-relative step behind it — out of the root. + # A descriptor names the inode that was checked, so there is nothing left to + # re-resolve and nothing left to swap. On a platform with no *at() family + # this is None and the per-component lstat floor applies instead, as it does + # throughout safe_fs. + try: + root_fd = open_directory_nofollow(root_path) + except ValueError as exc: + # Not a caller error: the root passed _resolve_root a moment ago, so + # this is the swap itself being refused. It belongs to the run, and + # nothing has been touched. + return ReconcileReport( + actions=[ + _run_error( + f"the skills root {root_path} could not be pinned for the " + f"reconcile: {exc}; no action was taken" + ) + ] ) - if manifest_error is None: - actions.extend(_rewrite_manifest(root_path, manifest, entries)) + try: + manifest, manifest_error = _load_manifest(root_path) + entries: dict[str, Any] = manifest.get("entries", {}) - return ReconcileReport(actions=actions) + actions: list[ReconcileAction] = [] + if manifest_error is not None: + # Run-level failure: there is no single skill key to hang it off. + actions.append(_run_error(manifest_error)) + + requests, incomplete = _resolve_requests(skills, deadline, on_unavailable) + + written, write_timed_out = _write_all( + root_path, root_fd, requests, entries, deadline + ) + actions.extend(written) + incomplete = incomplete or write_timed_out + + # Pruning is destructive, so it needs a trustworthy picture of both + # sides: a corrupt manifest means we do not know what we own, and an + # incomplete run — a retrieval that failed, or a deadline that expired + # mid-write — means we do not know what is still current. Either way, + # deleting would be a guess. + if prune and manifest_error is None and not incomplete: + actions.extend( + _prune( + root_path, + root_fd, + entries, + {request.key for request in requests}, + deadline, + ) + ) + + if manifest_error is None: + actions.extend(_rewrite_manifest(root_path, root_fd, manifest, entries)) + + return ReconcileReport(actions=actions) + finally: + if root_fd is not None: + os.close(root_fd) _RUN_LEVEL_KEY = "" @@ -240,6 +288,7 @@ def _run_error(message: str) -> ReconcileAction: def _write_all( root: Path, + root_fd: int | None, requests: list[_PendingWrite], entries: dict[str, Any], deadline: float, @@ -279,7 +328,7 @@ def _write_all( ) continue try: - actions.append(_write_one(root, request.skill, entries)) + actions.append(_write_one(root, root_fd, request.skill, entries)) except OSError as exc: # A safety net, not the primary defense. pathlib's stat probes swallow # only ENOENT/ENOTDIR/EBADF/ELOOP and re-raise every other errno, so an @@ -297,9 +346,19 @@ def _write_all( def _rewrite_manifest( - root: Path, manifest: dict[str, Any], entries: dict[str, Any] + root: Path, + root_fd: int | None, + manifest: dict[str, Any], + entries: dict[str, Any], ) -> list[ReconcileAction]: - """Writes the updated manifest. Returns an error action, or nothing.""" + """ + Writes the updated manifest. Returns an error action, or nothing. + + Relative to the root descriptor the caller has held all along, rather than + re-opening the root by path: this used to be the one operation that pinned + the root, and it pinned it too late to matter, since every skill file was + already written by the time it ran. + """ manifest["manifestVersion"] = MANIFEST_VERSION manifest["entries"] = entries try: @@ -308,7 +367,7 @@ def _rewrite_manifest( # planted field can raise RecursionError here — after every skill file is # already on disk. serialized = json.dumps(manifest, indent=2, sort_keys=True).encode("utf-8") - atomic_write_in(root, MANIFEST_FILENAME, serialized) + atomic_write(root, MANIFEST_FILENAME, serialized, dir_fd=root_fd) except Exception as exc: return [_run_error(f"the skills manifest could not be written: {exc}")] return [] @@ -508,6 +567,12 @@ def _resolve_root(root: str | os.PathLike[str]) -> Path: An unusable root is a caller error rather than a per-skill outcome, so this raises. Only the leaf directory is ever created — recursively creating missing ancestors would let a typo scatter a directory tree. + + What this establishes is that the root was usable *at this instant*, which + is a caller-error check and not a security boundary — the path it returns + can be replaced the moment it returns. ``write_skills`` pins the returned + path to a descriptor immediately afterwards, and that is what carries the + guarantee for the rest of the run. """ path = Path(os.fspath(root)) @@ -626,6 +691,13 @@ def _unsafe_path_reason( Shared by the write and prune paths: ``agents.md`` marks these checks non-relaxable, and maintaining them twice is how they drift. + These are defense in depth and not the boundary. Every one of them inspects + a path, so each is a check-then-use against anything that can rename a + component of that path — which is precisely why the destructive steps below + run relative to the root descriptor instead. They stay because they turn a + hostile layout into a reported refusal rather than a failed syscall, and + because they are the whole defense on a platform with no ``*at()`` family. + *require_directory* is the one genuine difference between the two callers. A write needs a real directory to write into. A prune only needs to not follow a link — an entry whose directory has been replaced by a plain file has @@ -686,7 +758,9 @@ def _key_rejection_reason(key: Any) -> str | None: return None -def _write_one(root: Path, skill: Skill, entries: dict[str, Any]) -> ReconcileAction: +def _write_one( + root: Path, root_fd: int | None, skill: Skill, entries: dict[str, Any] +) -> ReconcileAction: """Reconciles one verified skill against the managed root.""" key = skill.key @@ -724,7 +798,7 @@ def failed(message: str) -> ReconcileAction: # Sweep before writing rather than after, so a temp file this run is about # to create can never be a candidate. - _sweep_orphan_temp_files(root, key) + _sweep_orphan_temp_files(root, root_fd, key) # Overwrite only what the manifest records as ours under this key. entry = entries.get(relative) @@ -780,7 +854,9 @@ def failed(message: str) -> ReconcileAction: else: action = "written" - write_error = _write_through_descriptor(skill_dir, encoded, key, relative) + write_error = _write_through_descriptor( + skill_dir, encoded, key, relative, root_fd=root_fd + ) if write_error is not None: return failed(write_error) @@ -835,7 +911,7 @@ def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: os.close(fd) -def _sweep_orphan_temp_files(root: Path, key: str) -> None: +def _sweep_orphan_temp_files(root: Path, root_fd: int | None, key: str) -> None: """ Removes temp files a killed reconcile left behind under ``//``. @@ -852,7 +928,9 @@ def _sweep_orphan_temp_files(root: Path, key: str) -> None: its own temp naming for ``SKILL.md``, anchored at both ends, and asked of ``safe_fs`` rather than re-spelled here so the recognizer cannot drift from the writer; only regular files; and every removal relative to a descriptor - pinned with ``O_NOFOLLOW``. It never raises and never aborts the run: the + pinned with ``O_NOFOLLOW`` which is itself opened relative to the root + descriptor, so neither the root nor the skill directory can be swapped for + somewhere else to delete from. It never raises and never aborts the run: the reconcile itself has succeeded either way, so a sweep that cannot happen is a warning. """ @@ -863,7 +941,7 @@ def _sweep_orphan_temp_files(root: Path, key: str) -> None: return try: - with pinned_directory(skill_dir) as dir_fd: + with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: # Listing by path is safe even though the removals are # descriptor-relative: a name reaches the unlink only if it matches # the anchored temp pattern, and the unlink resolves it inside the @@ -900,7 +978,12 @@ def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> def _write_through_descriptor( - skill_dir: Path, encoded: bytes, key: str, relative: str + skill_dir: Path, + encoded: bytes, + key: str, + relative: str, + *, + root_fd: int | None, ) -> str | None: """ Performs the write itself. Returns a failure reason, or ``None`` on success. @@ -909,9 +992,14 @@ def _write_through_descriptor( write and this decides nothing: the directory is pinned to a descriptor and every remaining step is relative to it, so none of the checks above can be invalidated by a swap between here and the rename. + + The skill directory is created and opened relative to *root_fd*, so the + ``mkdir`` cannot be redirected either — ``mkdir`` follows a symlink at its + parent, which made a root swapped before this point enough to have the + directory, and then the file, created outside the root. """ try: - with pinned_directory(skill_dir, create=True) as dir_fd: + with pinned_directory(skill_dir, create=True, dir_fd=root_fd) as dir_fd: try: atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=dir_fd) except OSError as exc: @@ -970,7 +1058,11 @@ def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction def _prune( - root: Path, entries: dict[str, Any], requested: set[str], deadline: float + root: Path, + root_fd: int | None, + entries: dict[str, Any], + requested: set[str], + deadline: float, ) -> list[ReconcileAction]: """ Removes managed skills that are no longer requested. @@ -1019,7 +1111,7 @@ def _prune( continue try: - actions.append(_prune_one(root, relative, key, entries)) + actions.append(_prune_one(root, root_fd, relative, key, entries)) except OSError as exc: actions.append( _prune_error( @@ -1032,7 +1124,9 @@ def _prune( return actions -def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: +def _unlink_through_descriptor( + skill_dir: Path, relative: str, *, root_fd: int | None +) -> str | None: """ Performs the removal itself. Returns a failure reason, or ``None`` on success. @@ -1041,10 +1135,12 @@ def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: nothing. The directory is pinned before the unlink because unlink never follows a trailing symlink but does resolve the directory above it, so a ``/`` swapped for a symlink between the checks and here would - otherwise delete a file outside the root. + otherwise delete a file outside the root — and the pin is taken relative to + *root_fd* for the same reason one level up, since opening ``/`` + by path re-resolves ```` and a swap there redirects the whole removal. """ try: - with pinned_directory(skill_dir) as dir_fd: + with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: try: unlink_file(skill_dir, SKILL_FILENAME, dir_fd=dir_fd) except SymlinkRefused: @@ -1057,7 +1153,11 @@ def _unlink_through_descriptor(skill_dir: Path, relative: str) -> str | None: def _prune_one( - root: Path, relative: str, key: str, entries: dict[str, Any] + root: Path, + root_fd: int | None, + relative: str, + key: str, + entries: dict[str, Any], ) -> ReconcileAction: """Removes one managed skill file, and its directory when that empties it.""" skill_dir = root / key @@ -1070,19 +1170,24 @@ def _prune_one( # Before the removal, so the ``rmdir`` below is not defeated by an orphaned # temp file that nothing else on disk records. - _sweep_orphan_temp_files(root, key) + _sweep_orphan_temp_files(root, root_fd, key) removed_from_disk = False if target.exists(): - failure = _unlink_through_descriptor(skill_dir, relative) + failure = _unlink_through_descriptor(skill_dir, relative, root_fd=root_fd) if failure is not None: return _prune_error(key, failure, version) removed_from_disk = True try: - # Path-based, and safe that way: rmdir never follows a trailing + # Relative to the root descriptor. rmdir never follows a trailing # symlink (it fails ENOTDIR) and only ever succeeds on an empty - # directory. - skill_dir.rmdir() + # directory, so the *key* was never the exposure here — the root + # above it was, since a path-based rmdir re-resolves it and would + # remove an attacker-chosen directory that happened to be empty. + if root_fd is not None: + os.rmdir(key, dir_fd=root_fd) + else: + skill_dir.rmdir() except OSError: pass # the customer keeps their own files here too diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 812eff72..bd891b1f 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -216,13 +216,25 @@ class _SwapRootDuring: (``/.claude/skills``) that parent is ``.claude``, which the agent identity typically owns. - Fires once, on the first call whose path argument is ``/``; every - other call passes straight through. + Fires once, on the first call naming the skill directory — as the absolute + ``/`` or as the bare ```` — and every other call passes + straight through. + + Matching *both* spellings is what keeps this honest across the fix. Code + that opens ``/`` by path names it absolutely; code that opens it + relative to a held root descriptor passes the bare key. Triggering only on + the absolute form would mean that the moment the root is pinned the trigger + stops matching, the swap never fires, and all three tests below pass while + asserting nothing — the ``race.swapped is True`` guard would be the only + thing standing between a real fix and a vacuous one, and it would be + load-bearing for the wrong reason. Firing in both worlds is what makes these + tests fail before the fix and pass after it. """ def __init__(self, attribute: str, root: Path, key: str, outside: Path) -> None: self.attribute = attribute self.root = Path(os.path.realpath(root)) + self.key = key self.trigger = self.root / key self.moved_to = self.root.parent / f"{self.root.name}.real" self.outside = outside @@ -230,17 +242,15 @@ def __init__(self, attribute: str, root: Path, key: str, outside: Path) -> None: self._real = getattr(os, attribute) def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: - # ``os.mkdir(path, mode)`` and ``os.open(path, flags, ...)`` both take the - # path first; descriptor-relative calls pass a bare name, which never - # compares equal to the absolute trigger. - if ( - not self.swapped - and isinstance(first, (str, os.PathLike)) - and Path(os.fspath(first)) == self.trigger - ): - os.rename(self.root, self.moved_to) - os.symlink(self.outside, self.root, target_is_directory=True) - self.swapped = True + # ``os.mkdir(path, mode)`` and ``os.open(path, flags, ...)`` both take + # the path first, whether that path is absolute or a bare name resolved + # against a ``dir_fd``. + if not self.swapped and isinstance(first, (str, os.PathLike)): + named = os.fspath(first) + if named == self.key or Path(named) == self.trigger: + os.rename(self.root, self.moved_to) + os.symlink(self.outside, self.root, target_is_directory=True) + self.swapped = True return self._real(first, *args, **kwargs) def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootDuring: @@ -1160,23 +1170,29 @@ async def test_directory_swapped_at_the_prune_cannot_redirect_the_unlink( class TestRootSwapRaces: """The swap one level up: the managed *root*, not ``/``. - ``_resolve_root`` validates the root once and returns a path; nothing holds - it open. Each write and each prune then opens ``/`` *by path* - with ``O_NOFOLLOW | O_DIRECTORY`` and pins that. ``O_NOFOLLOW`` guards only - the final component, so the root and every ancestor are re-resolved on - every such open, and a root swapped for a symlink after validation - redirects the open — and with it every descriptor-relative step behind it - — into the attacker's directory. ``atomic_write_in`` does refuse the - manifest rewrite afterwards, but by then the skill file is already outside - the root. - - ``TestSymlinkAttacks`` proves the ``/`` swap is closed. These - three state the same contract for the root — nothing lands outside it, no - outside file is overwritten, no outside file is removed. The code does not - yet meet it, so they fail, and the red run is the demonstration. The fix is - to pin the root once at the top of ``write_skills`` and walk from it: - ``os.mkdir(key, dir_fd=root_fd)``, ``os.open(key, ..., dir_fd=root_fd)``, - and ``atomic_write(..., dir_fd=root_fd)`` for the manifest. + ``_resolve_root`` validates the root and returns a path. It used to end + there: each write and each prune then opened ``/`` *by path* with + ``O_NOFOLLOW | O_DIRECTORY`` and pinned that, and ``O_NOFOLLOW`` guards only + the final component — so the root and every ancestor were re-resolved on + every such open, and a root renamed aside and replaced with a symlink after + validation redirected the open, and with it every descriptor-relative step + behind it, into the attacker's directory. The manifest rewrite refused, but + by then the skill file was already outside the root. + + ``write_skills`` now pins the root once, before anything is read or written, + and holds that descriptor for the whole reconcile: the skill directory is + created and opened relative to it, the unlink and the ``rmdir`` run relative + to it, and so does the manifest write. The swap still happens in each of + these — ``race.swapped`` asserts it did — and the descriptor still names the + directory that was validated, so the reconcile carries on inside the real + root, which the rename has moved to ``race.moved_to``. + + ``TestSymlinkAttacks`` proves the same for a swapped ``/``. The + contract here is the root's: nothing lands outside it, no outside file is + overwritten, no outside file is removed. Note what each assertion permits — + either the reconcile refused outright, or it completed against the real + root — because *which* of those happens depends on where in the sequence + the swap lands, and neither is an escape. """ @_needs_dir_fd @@ -1253,6 +1269,107 @@ async def test_root_swapped_at_the_prune_cannot_redirect_the_unlink( assert victim.exists() and victim.read_text(encoding="utf-8") == "precious\n" assert report.ok is False or not (race.moved_to / "a" / "SKILL.md").exists() + @_needs_dir_fd + async def test_a_root_swapped_before_the_pin_is_refused_at_the_run_level( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The other side of the window the three above race into. + + Those swap the root *after* it is pinned, and the held descriptor is + what carries the reconcile through to the real directory. This one swaps + it in the only interval left — after ``_resolve_root`` has validated the + root and before ``write_skills`` opens it — so there is no descriptor + yet to fall back on. The ``O_NOFOLLOW`` open of the root is what has to + refuse it, and because the root passed validation a moment earlier this + is a run-level error rather than the ``ValueError`` an unusable root + raises: nothing has been touched, and the report says so. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + victim = outside / "a" / "SKILL.md" + victim.write_text("precious\n", encoding="utf-8") + moved_to = tmp_path / "skills.real" + + real_resolve_root = skills_fs_module._resolve_root + + def resolve_then_swap(argument: Any) -> Path: + resolved = real_resolve_root(argument) + os.rename(root, moved_to) + os.symlink(outside, root, target_is_directory=True) + return resolved + + monkeypatch.setattr(skills_fs_module, "_resolve_root", resolve_then_swap) + + report = await write_skills([_skill("a")], root) + + assert report.ok is False + assert [action.key for action in report.actions] == [""], _error_messages( + report + ) + assert "could not be pinned" in (report.actions[0].error or "") + # Neither written into nor read as a managed root. + assert victim.read_text(encoding="utf-8") == "precious\n" + assert not (outside / "a" / MANIFEST_NAME).exists() + assert list(moved_to.iterdir()) == [] + + @_needs_dir_fd + async def test_every_destructive_call_runs_relative_to_a_descriptor( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The invariant behind the three races, asserted directly. + + Each test above plants one swap and checks the blast radius, so each + covers the one sequence it races. This covers the property they are + each an instance of: across a reconcile that creates a directory, + writes a file, renames a temp over it, writes the manifest, unlinks and + removes the directory, *no* destructive call names an absolute path. + Every one passes a bare component and a descriptor to resolve it + against, which is what makes the swap unable to redirect any of them. + + Written as an audit rather than another race because the failure mode is + a new call site, not a new attack: one operation added on the full path + would reopen the window for that operation alone, and no swap test aimed + at the existing sequences would notice. + """ + destructive = ("mkdir", "rmdir", "unlink", "replace") + recorded: list[tuple[str, str, bool]] = [] + + def recorder(name: str) -> Any: + real = getattr(os, name) + + def wrapper(first: Any, *args: Any, **kwargs: Any) -> Any: + recorded.append( + ( + name, + os.fspath(first) + if isinstance(first, (str, os.PathLike)) + else repr(first), + # replace takes src_dir_fd/dst_dir_fd rather than dir_fd. + any("dir_fd" in keyword for keyword in kwargs), + ) + ) + return real(first, *args, **kwargs) + + return wrapper + + for name in destructive: + monkeypatch.setattr(safe_fs_module.os, name, recorder(name)) + + written = await write_skills([_skill("a")], root) + pruned = await write_skills([], root) + + assert written.ok is True, _error_messages(written) + assert pruned.ok is True, _error_messages(pruned) + # mkdir, replace (the skill file), replace (the manifest), unlink, + # rmdir, replace (the manifest again) — the sequence must have run, or + # the audit below is vacuous. + assert {entry[0] for entry in recorded} == set(destructive), recorded + assert [ + entry for entry in recorded if os.path.isabs(entry[1]) or not entry[2] + ] == [] + class TestWithoutDirFd: """The full-path fallback for platforms with no ``*at()`` family. From 50409422aa825b5cb3d150486010c874bd6dcf81 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 4 Sep 2026 15:23:02 -0400 Subject: [PATCH 14/43] docs(client): deny the agent write access to the root's ancestors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The privilege-separation checklist denied the agent identity the managed root, the per-skill directories, the files and the manifest, and said nothing about the root's parent. That was the precondition for the SEC-8985 row 2 root swap: renaming any ancestor is what lets the root be replaced with a symlink, and in the documented `/.claude/skills` layout the parent is `.claude`, which an agent identity is otherwise likely to own outright. The checklist and its shell snippet now walk every ancestor up to `/`. Write access to one of them is a strictly larger capability than racing the reconcile — no timing is involved, it persists until someone notices, and descriptor pinning inside `write_skills` cannot address it, because the substituted tree is what the agent reads rather than what the SDK wrote. The platform-bound paragraph claimed a descriptor held for the whole reconcile before that was true of the root; it now describes what is actually held and for how long, and names the root's ancestors alongside the root as the security boundary on every platform. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index bc8df592..1602264b 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -411,18 +411,24 @@ never writes through a symlink; writes are atomic (temp file, `fsync`, rename) a `0644`; and if the manifest is unreadable it performs no destructive action at all. Removing a skill from a variation is how revocation works — the next reconcile prunes it. -**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX every destructive -step — the open, the rename, the unlink — runs relative to a directory descriptor opened -`O_RDONLY|O_DIRECTORY|O_NOFOLLOW` and held for the whole reconcile, so a directory swapped for -a symlink *after* its checks cannot redirect a write or a delete: the descriptor names the -inode that was checked, which closes the swap window rather than narrowing it. Windows has no +**Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX the managed root +is opened once per reconcile — `O_RDONLY|O_DIRECTORY|O_NOFOLLOW`, checked with `fstat`, and +held until the call returns — and every destructive step under it runs relative to that +descriptor: the per-skill directory is created and opened relative to the root, the unlink and +the `rmdir` relative to the directory, and the manifest write relative to the root. So a +directory swapped for a symlink *after* its checks cannot redirect a write or a delete: the +descriptor names the inode that was checked, which closes the swap window rather than +narrowing it. Because the root itself is pinned this holds for the root and its ancestors too, +not only for `/` — but only from the instant the root is opened, which is why the +checklist below denies the agent write access to every ancestor. Windows has no `*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check taken immediately before each step. That floor is a check-then-use race rather than a closed window: an attacker who already holds **write permission on the managed root** can still win it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are deliberately not implemented in this release, and Windows is not a tested platform for it — -neither SDK repository has a Windows CI runner. Treat write permission on the managed root as -the security boundary on every platform, and on Windows as the *only* one. +neither SDK repository has a Windows CI runner. Treat write permission on the managed root +**or on any directory above it** as the security boundary on every platform, and on Windows as +the *only* one. **One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is @@ -510,7 +516,12 @@ they describe a directory the agent can freely rewrite. - the managed root itself, - the per-skill directories `//` and the files `//SKILL.md`, -- the manifest at `/.launchdarkly-skills.json`. +- the manifest at `/.launchdarkly-skills.json`, +- **the root's parent, and every ancestor directory above it.** Write access to an ancestor is + write access to the root by another route: it permits renaming the root aside and leaving a + symlink in its place, which redirects the reconcile — and the agent's own skill lookups — at + a directory the agent controls. In the documented layout `/.claude/skills` that parent + is `.claude`, which an agent identity is otherwise likely to own outright. ```bash # Run as the agent's user. Every line should print DENIED. @@ -519,6 +530,15 @@ for target in "$root" "$root/.launchdarkly-skills.json" "$root"/*/ "$root"/*/SKI [ -e "$target" ] || continue if [ -w "$target" ]; then echo "WRITABLE — fix this: $target"; else echo "DENIED: $target"; fi done + +# The root's ancestors, up to /. A writable one is enough to rename the root +# aside and put a symlink where it was, so these matter as much as the root. +ancestor=$(cd "$(dirname "$root")" && pwd) +while :; do + if [ -w "$ancestor" ]; then echo "WRITABLE ANCESTOR — fix this: $ancestor"; else echo "DENIED: $ancestor"; fi + [ "$ancestor" = / ] && break + ancestor=$(dirname "$ancestor") +done ``` Note that the managed root's own mode is **yours, not the SDK's**: `write_skills` creates only @@ -536,6 +556,13 @@ touch. `write_skills` re-validates every manifest entry from scratch for exactly it treats that file as untrusted input, never as authorization — but an agent that cannot edit it at all is the stronger position, and only your deployment can provide that. +The same reasoning is why the ancestors are on the list. An identity that can rename a +directory above the root does not need write access to anything inside it: it can substitute +the whole tree, and everything the agent then loads as a skill is a file it wrote itself. That +is a different capability from racing the reconcile — no timing is involved, and it persists +until someone notices — and no amount of descriptor pinning inside `write_skills` addresses +it, because the substituted tree is what the agent reads, not what the SDK wrote. + **The SDK deliberately does not report whether the root is writable.** There is no such field on `ReconcileReport`, and its absence is a decision rather than an oversight. The SDK knows only its own identity, which trivially has write access — it just wrote there. It cannot know From dd2d55cd680fe5cd56bc937804797cb6943ecdd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:02:00 -0400 Subject: [PATCH 15/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20re-reconcile=20on=20delivery=20with=20watch=5Fskill?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_skills is a one-shot reconcile, so a revocation takes effect at the next process restart. watch_skills runs that reconcile now and again whenever the configured store reports a change, so a revoked skill's files leave the disk within a debounce interval of the store learning about it. It is wired to the SkillStore interface, not to any one transport: it needs a store that implements add_listener and nothing more, and refuses loudly when the store does not, since a watcher that silently never fires looks exactly like one whose skills never changed. Above the interface, remove_listener joins add_listener as the optional second half of change notification, on the SkillStore contract and on InMemorySkillStore. SkillWatcher.close needs it to detach; without it a store held every watcher ever created for the rest of its life. The watcher probes for it, so a store without it keeps working. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 3 +- packages/client/agents.md | 11 +- .../src/launchdarkly_ai_server/__init__.py | 4 + .../src/launchdarkly_ai_server/skills.py | 16 + .../src/launchdarkly_ai_server/skills_core.py | 16 +- .../launchdarkly_ai_server/skills_watch.py | 292 ++++++++++++++++++ packages/client/tests/test_skills.py | 29 ++ packages/client/tests/test_skills_watch.py | 178 +++++++++++ 8 files changed, 538 insertions(+), 11 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_watch.py create mode 100644 packages/client/tests/test_skills_watch.py diff --git a/packages/client/README.md b/packages/client/README.md index bc8df592..41d5c6b3 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -458,8 +458,9 @@ Windows. | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | -| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)`. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 2558f9c3..ca56b80e 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | @@ -198,11 +199,11 @@ Three layers, in increasing order of blast radius: ### The store seam, and why version is part of the lookup -`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and an optional -`add_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied -to the answer, and that is load-bearing: a delivery payload carries the newest version of -every skill *plus* every version any variation currently pins, so two versions of one key -coexist routinely. A seam keyed by key alone would answer a pinned reference with the newest +`SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and the optional +pair `add_listener(kind, fn)` / `remove_listener(kind, fn)`. Version is part of the **lookup +identity**, not a filter applied to the answer, and that is load-bearing: a delivery payload +carries the newest version of every skill *plus* every version any variation currently pins, +so two versions of one key coexist routinely. A store keyed by key alone would answer a pinned reference with the newest object, and the caller would then have to reject it — turning the primary use case, a version-pinned attachment, into a missing skill. `version=None` asks for the newest held. diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 753e1e95..f863d847 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -53,6 +53,7 @@ OnUnavailable, write_skills, ) +from .skills_watch import SkillWatcher, watch_skills from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -237,6 +238,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", + # skills — the eager re-reconcile + "watch_skills", + "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name "ReconcileActionKind", "OnUnavailable", diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index cb21986b..9016ab9f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -167,6 +167,22 @@ def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: """ self._listeners.setdefault(kind, []).append(fn) + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*, so a subsequent ``put`` no longer calls it. + + Removes one occurrence: a callable registered twice must be removed twice. + Removing a callable that is not registered is a no-op, not an error, so a + consumer that detaches on close can do so unconditionally. + """ + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + # --------------------------------------------------------------------------- # Reference discovery diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 74cb706b..22166799 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -151,11 +151,17 @@ class SkillStore(Protocol): Duck-typed on purpose, mirroring how the LaunchDarkly client interface works in this package: pass any object carrying these methods. - ``add_listener(kind, fn)`` is part of the seam but - **optional**, which is why it is deliberately not declared here: a Protocol - member is required for structural compatibility, so declaring it would reject - every store that does not implement it. Nothing in this module calls it — it - exists for the delivery transport to push updates through. + ``add_listener(kind, fn)`` and ``remove_listener(kind, fn)`` are part of the + interface but **optional**, which is why they are deliberately not declared + here: a Protocol member is required for structural compatibility, so declaring + them would reject every store that does not implement them. Nothing in this + module calls either — they exist for the delivery transport to push updates + through, and for a consumer such as ``watch_skills`` to stop receiving them. + A store that implements ``add_listener`` should implement ``remove_listener`` + too; consumers probe for it and skip detaching when it is absent, so an + older store keeps working at the cost of a listener that lives as long as + the store does. ``remove_listener`` removes one occurrence of *fn* under + *kind* and is a no-op when *fn* is not registered. The raw objects a store serves are wire-shaped, with camelCase field names identical across language implementations:: diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py new file mode 100644 index 00000000..05263057 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -0,0 +1,292 @@ +""" +Agent Skills — re-reconcile on delivery, so revocation does not wait for a restart. + +``write_skills`` is a one-shot reconcile: it materializes what the store holds +now. With a hand-populated store that is sufficient, and a revocation takes +effect at the next process restart. + +A streaming FDv2 connection changes the premise. A ``delete-object`` reaches a +live connection in **seconds**, and the store publishes a change listener, so +wiring the two together collapses the gap between "LaunchDarkly revoked this +skill" and "its ``SKILL.md`` is off the agent's disk" from a process lifetime to +a debounce interval. + +``on_unavailable="keep"`` stays the default: an outage must not read as +"everything was revoked". A watcher that pruned on a failed retrieval would +convert every transport failure into deletion of a customer's skill files. + +Layering: this module sits *above* ``skills_fs`` and calls ``write_skills`` +without modifying it. Nothing in the reconcile, the accessors, or verification +knows this file exists. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from collections.abc import Callable, Sequence +from typing import Any + +from .skills_core import SKILL_OBJECT_KIND, get_store +from .skills_fs import OnUnavailable, write_skills +from .types import ReconcileReport, Skill, SkillReference + +logger = logging.getLogger(__name__) + +DEFAULT_DEBOUNCE_SECONDS = 0.5 +""" +How long a change waits for its neighbours before a reconcile runs. + +A full payload transfer commits many objects at once and the listener fires per +object, so without coalescing a payload of forty skills would run forty +reconciles against one root. Half a second is far below the seconds-scale +latency this feature is trying to achieve and far above the microseconds a +commit's listener calls take. +""" + + +class SkillWatcher: + """ + A running re-reconcile. Returned by ``watch_skills``; stop it with ``close``. + + One watcher owns one root. **Do not point two watchers at the same root**, + and do not run ``write_skills`` against a watched root concurrently: the + reconcile's own contract is one root, one reconcile at a time, because two + interleaved runs lose the loser's manifest entries and leave the files it + wrote unmanaged. This class enforces that for its *own* reconciles — they run + on a single worker thread, serialised — and cannot enforce it against a + caller who reconciles the same root by hand. + + The watcher owns its registration on *store*: it registers ``notify`` when + constructed and unregisters it in ``close``, so a closed watcher is no longer + reachable from the store and can be collected. *store* must implement + ``add_listener``; ``remove_listener`` is probed for and, when the store does + not offer it, the listener stays registered for the store's lifetime. + """ + + def __init__( + self, + request: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + store: Any, + *, + prune: bool, + timeout: float, + on_unavailable: OnUnavailable, + debounce: float, + on_reconcile: Callable[[ReconcileReport], Any] | None, + ) -> None: + self._request = request + self._root = root + self._prune = prune + self._timeout = timeout + self._on_unavailable = on_unavailable + self._debounce = debounce + self._on_reconcile = on_reconcile + + self._wake = threading.Event() + self._stop = threading.Event() + self._reconciles = 0 + self._lock = threading.Lock() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-reconcile", daemon=True + ) + + # Register before starting the worker: ``notify`` only sets an event, so a + # change that lands in between is picked up as soon as the worker runs, + # and a store whose ``add_listener`` raises leaves no thread behind. + self._store = store + self._registered = False + store.add_listener(SKILL_OBJECT_KIND, self.notify) + self._registered = True + self._thread.start() + + # -- the listener the store calls ------------------------------------- + + def notify(self, _raw: Any = None) -> None: + """ + The store's change listener. Records that something changed; runs nothing. + + Deliberately trivial. It is called on the delivery thread, where a + reconcile — which does synchronous filesystem I/O, an fsync per file, and + a manifest rewrite — would stall event processing for the duration and, + on a stream, let the connection's read buffer back up behind a disk write. + The argument is ignored: a put's raw object and a revocation's tombstone + both mean the same thing here, which is "the store is not what it was". + """ + self._wake.set() + + # -- the worker -------------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + if not self._wake.wait(timeout=0.5): + continue + if self._stop.is_set(): + return + # Coalesce the rest of the burst. Clearing *before* the sleep rather + # than after is what makes a change arriving mid-debounce trigger the + # next pass instead of being swallowed by this one. + self._wake.clear() + if self._stop.wait(self._debounce): + return + self._reconcile_once() + + def _reconcile_once(self) -> None: + try: + report = asyncio.run( + write_skills( + self._request, + self._root, + prune=self._prune, + timeout=self._timeout, + on_unavailable=self._on_unavailable, + ) + ) + except Exception: + # A watcher that died on one bad reconcile would silently stop + # tracking revocations, which is worse than a noisy one. + logger.error( + "A skill re-reconcile raised; the watcher continues", exc_info=True + ) + return + + with self._lock: + self._reconciles += 1 + changed = [ + action + for action in report.actions + if action.action in ("written", "updated", "removed", "error") + ] + if changed: + logger.info( + "Re-reconciled skills after a delivery change: %d action(s) of note", + len(changed), + ) + if self._on_reconcile is not None: + try: + self._on_reconcile(report) + except Exception: + logger.error("A watch_skills callback raised", exc_info=True) + + # -- lifecycle --------------------------------------------------------- + + @property + def reconciles(self) -> int: + """How many re-reconciles have completed since the watcher started. + + Excludes the initial reconcile ``watch_skills`` awaits, which is the + caller's own result.""" + with self._lock: + return self._reconciles + + def close(self, timeout: float = 15.0) -> None: + """ + Stops watching. Idempotent. Does not undo anything already on disk. + + Waits out an in-flight reconcile rather than interrupting one, because a + reconcile killed between its content writes and its manifest rewrite is + the one case the manifest format has to recover from — worth avoiding when + we control the timing. + + Detaches ``notify`` from the store first, so no further change reaches a + watcher that is shutting down and the store no longer holds a reference to + it. A store without the optional ``remove_listener`` is left as it is + rather than failing the close. + """ + self._detach() + self._stop.set() + self._wake.set() + if self._thread.is_alive() and self._thread is not threading.current_thread(): + self._thread.join(timeout=timeout) + + def _detach(self) -> None: + with self._lock: + if not self._registered: + return + self._registered = False + remove_listener = getattr(self._store, "remove_listener", None) + if callable(remove_listener): + remove_listener(SKILL_OBJECT_KIND, self.notify) + + def __enter__(self) -> SkillWatcher: + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +async def watch_skills( + skills: Sequence[Skill | SkillReference | str] | str, + root: str | os.PathLike[str], + *, + prune: bool = True, + timeout: float = 10.0, + on_unavailable: OnUnavailable = "keep", + debounce: float = DEFAULT_DEBOUNCE_SECONDS, + on_reconcile: Callable[[ReconcileReport], Any] | None = None, +) -> tuple[ReconcileReport, SkillWatcher]: + """ + Reconciles now, then re-reconciles whenever delivery changes. + + Every argument that ``write_skills`` takes means the same thing here and is + passed straight through; the reconcile's semantics are untouched. Returns the + initial reconcile's report — so a caller can fail fast on a bad root or a + corrupt manifest exactly as they would with ``write_skills`` — paired with a + ``SkillWatcher`` to close when the process is done:: + + report, watcher = await watch_skills("*", "/etc/agent/skills") + try: + ... + finally: + watcher.close() + + A revocation delivered over a streaming connection then prunes the skill's + files within ``debounce`` of arriving, rather than at the next restart. + + Requires a store that implements the optional ``add_listener`` half of the + ``SkillStore`` interface. Raises ``RuntimeError`` when no store is configured, + and when the configured store has no ``add_listener`` — the second case + failing loudly rather than degrading to a one-shot reconcile, because a + watcher that silently never fires looks exactly like a watcher whose skills + never changed. The optional ``remove_listener`` lets ``SkillWatcher.close`` + detach from the store; a store without it still works, but each closed + watcher then stays registered for the store's lifetime. + """ + store = get_store() + if store is None: + raise RuntimeError( + "watch_skills needs a configured skill store. Configure one with " + 'init_client(options={"skillStore": store}).' + ) + add_listener = getattr(store, "add_listener", None) + if not callable(add_listener): + raise RuntimeError( + "watch_skills needs a skill store that implements add_listener(kind, " + "fn); the configured store does not, so delivery changes cannot be " + "observed. Use write_skills for a one-shot reconcile, or configure a " + "store with a delivery transport." + ) + if debounce < 0: + raise ValueError(f"debounce must not be negative, got {debounce!r}") + + # The initial reconcile runs first and on the caller's thread, so its report + # is the caller's to inspect and a bad root raises out of `watch_skills` + # rather than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + + watcher = SkillWatcher( + skills, + root, + store, + prune=prune, + timeout=timeout, + on_unavailable=on_unavailable, + debounce=debounce, + on_reconcile=on_reconcile, + ) + return report, watcher diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 6470f0c0..d3f83095 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -548,6 +548,35 @@ def test_put_does_not_notify_other_kind_listeners( assert seen == [] + def test_remove_listener_stops_put_notifying_it(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert seen == [] + + def test_remove_listener_removes_one_occurrence(self, make_raw_skill: Any) -> None: + s = InMemorySkillStore() + seen: list[dict[str, Any]] = [] + s.add_listener("skill", seen.append) + s.add_listener("skill", seen.append) + s.remove_listener("skill", seen.append) + + s.put(make_raw_skill(key="a")) + + assert len(seen) == 1 + + def test_remove_listener_of_an_unregistered_callable_is_a_no_op(self) -> None: + s = InMemorySkillStore() + s.remove_listener("skill", print) + s.add_listener("skill", print) + s.remove_listener("flag", print) + s.remove_listener("skill", print) + s.remove_listener("skill", print) + class TestStoreConfiguration: """Store wiring on the lifecycle layer.""" diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py new file mode 100644 index 00000000..48a95f07 --- /dev/null +++ b/packages/client/tests/test_skills_watch.py @@ -0,0 +1,178 @@ +""" +Tests for ``watch_skills`` / ``SkillWatcher`` — the eager re-reconcile. + +The watcher is wired to the ``SkillStore`` interface, not to any one transport: +it needs a store that implements ``add_listener``, and nothing more. These tests +therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its +listeners synchronously, and from small hand-written store doubles. + +Every test writes only inside pytest's ``tmp_path``. The watcher runs a real +worker thread, so tests wait on observable outcomes rather than on fixed sleeps +wherever the outcome is something that *does* happen; a fixed sleep is used only +to assert that something does *not*. +""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore, init_client, watch_skills +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + +# --------------------------------------------------------------------------- +# Starting a watch, and what it refuses +# --------------------------------------------------------------------------- + + +class TestWatchSkills: + async def test_the_in_memory_store_can_also_drive_a_watch( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + """The watcher is wired to the ``SkillStore`` interface, not to the FDv2 + store.""" + store = InMemorySkillStore() + store.put(make_raw_skill(key="a", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "a" / "SKILL.md" + assert written.read_text() == "body" + store.put(make_raw_skill(key="a", version=2, content="new body")) + assert wait_until(lambda: written.read_text() == "new body", timeout=10) + finally: + watcher.close() + + async def test_a_burst_of_changes_coalesces_into_few_reconciles( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.1) + try: + for i in range(12): + store.put(make_raw_skill(key=f"skill-{i}")) + time.sleep(0.5) + # Twelve objects put back to back fire twelve listener calls; without + # coalescing that is twelve reconciles of one root. + assert watcher.reconciles <= 2 + finally: + watcher.close() + + async def test_a_store_with_no_listener_support_is_refused_loudly( + self, tmp_path: Any + ) -> None: + class NoListeners: + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + await init_client(options={"skillStore": NoListeners()}, client=object()) + with pytest.raises(RuntimeError, match="add_listener"): + await watch_skills("*", tmp_path / "s") + + async def test_no_store_configured_raises(self, tmp_path: Any) -> None: + with pytest.raises(RuntimeError, match="configured skill store"): + await watch_skills("*", tmp_path / "s") + + +# --------------------------------------------------------------------------- +# Closing a watch +# --------------------------------------------------------------------------- + + +class TestWatcherDetachesOnClose: + """``SkillWatcher.close`` unregisters ``notify``, so a closed watcher is + neither called nor kept alive by the store.""" + + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + async def test_a_closed_watcher_is_no_longer_notified( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + store = InMemorySkillStore() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="first")) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert watcher.notify in self._skill_listeners(store) + + watcher.close() + + assert watcher.notify not in self._skill_listeners(store) + store.put(make_raw_skill(key="pdf-extraction", version=4, content="second")) + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction", 4) is not None + ), + timeout=10, + ) + time.sleep(0.3) + assert written.read_text() == "first" + assert watcher.reconciles == 0 + + async def test_close_twice_does_not_raise(self, tmp_path: Any) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + watcher.close() + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_repeated_watchers_leave_no_listeners_behind( + self, tmp_path: Any + ) -> None: + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + for _ in range(5): + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert len(self._skill_listeners(store)) == 1 + watcher.close() + assert self._skill_listeners(store) == [] + + async def test_a_store_without_remove_listener_still_closes( + self, tmp_path: Any + ) -> None: + """``remove_listener`` is optional: an older store keeps working, at the + cost of the listener staying registered.""" + + class AddOnly: + def __init__(self) -> None: + self.listeners: list[Any] = [] + + def get_object(self, *_a: Any, **_k: Any) -> None: + return None + + def all_objects(self, _kind: str) -> dict[str, Any]: + return {} + + def add_listener(self, _kind: str, fn: Any) -> None: + self.listeners.append(fn) + + store = AddOnly() + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + assert store.listeners == [watcher.notify] + + watcher.close() + watcher.close() + + assert store.listeners == [watcher.notify] From 0f797c0b167adf8be462ae8e8b2431705d75dfd7 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:03:21 -0400 Subject: [PATCH 16/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20protocol,=20without=20the?= =?UTF-8?q?=20network?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The half of the delivery transport that has no I/O: identifying a skill object on the wire by kind inline-resource plus category skill, translating it into the raw object shape the SkillStore interface defines, holding it by (key, objectVersion), and applying a payload's events as one commit at payload-transferred. The store that puts a connection underneath this follows separately, so the three decisions that matter most can be reviewed on their own: - objectVersion is the skill's version; version is the payload's. The translation happens in one place and TestVersionTranslation asserts it in both directions, because confusing them fails silently. - Changes commit at payload-transferred, not per object. A half-applied full transfer would briefly empty the store, which with pruning on is the difference between a reconcile and deleting a customer's files. - A hashless object is held, not dropped, so verification withholds it with a reason code rather than the transport reporting it absent. Flag and segment objects share the connection and are skipped and counted, not rejected. Nothing here is exported yet; the store exports it. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++ .../src/launchdarkly_ai_server/skills_fdv2.py | 601 +++++++++++++++++ packages/client/tests/test_skills_fdv2.py | 605 ++++++++++++++++++ 3 files changed, 1252 insertions(+) create mode 100644 packages/client/src/launchdarkly_ai_server/skills_fdv2.py create mode 100644 packages/client/tests/test_skills_fdv2.py diff --git a/packages/client/agents.md b/packages/client/agents.md index ca56b80e..80ec0538 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,6 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -218,6 +219,51 @@ one place that collapses the result to one object per key, because both whole-st consumers need it — `all_skills`, since a list holding two versions of one key is not a set of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +### The delivery transport, and the one field that will bite you + +`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the +shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above +that interface knows it exists**. If a transport change ever seems to require editing an +accessor, verification, or `write_skills`, the adapter boundary is wrong. + +**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill +`put-object` carries both, and they are not interchangeable: + +```json +{"key":"pdf-extraction","kind":"inline-resource","category":"skill", + "objectVersion":3,"version":42, + "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} +``` + +`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored +`version`. `version` (42) is the version of the *payload* the object arrived in — it moves +when anything in the environment moves, including a flag with nothing to do with skills. +Reading it as the skill's version fails **silently**: the object verifies, the hash matches, +and the caller gets content under a version number that means nothing. Flags and segments +carry only `version` and omit both `category` and `objectVersion`, which is exactly why the +two fields look interchangeable. `_store_object_from_put` is the only place the translation +happens, and `TestVersionTranslation` asserts it in both directions. + +**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else +is ignored, not rejected.** An environment's payload assignment carries its flag payload +alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. +Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect +loop — a flag-delivery outage caused by a skills rollout. + +**Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the +unit of consistency: a half-applied full transfer would publish a state the server never +described, and would briefly empty the store — which, with pruning on, is the difference +between a reconcile and deleting a customer's skill files. An interrupted transfer therefore +leaves last known good intact, and listeners fire once per commit. + +**A hashless object is held, not dropped.** Verification withholds it with +`missing_content_hash`; the transport's job is to make that loud (an error per object, a +summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work +around it. Dropping it at the transport would report `absent` — indistinguishable from "no +such skill" — and would let a prune delete the last known-good copy on disk. Never synthesize +a hash from the delivered content: that certifies the content against itself and verifies +nothing. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py new file mode 100644 index 00000000..80ef1224 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -0,0 +1,601 @@ +""" +Agent Skills — the FDv2 delivery protocol. + +The half of the delivery transport that has no I/O: identifying skill objects +on the wire, translating them into the raw object shape the ``SkillStore`` +interface defines, holding them by ``(key, objectVersion)``, and applying a +payload's events as one consistent commit. ``FDv2SkillStore``, the store that +puts a network connection underneath this, follows in a separate change. + +It sits *below* the ``SkillStore`` interface, and everything above — the +accessors, integrity verification, the ``Skill`` dataclass, materialization — +is unaware of it. + +Layering:: + + launchdarkly_ai_server + └─ SkillStore protocol (skills_core) ── the interface accessors call + └─ FDv2SkillStore (this module) ── deserialise, hold, serve + └─ LaunchDarkly's SDK-facing FDv2 channel + GET /sdk/poll, GET /sdk/stream, authenticated with the + environment's server-side SDK key + +Dependencies run one way: this module imports nothing from the feature beyond +the version validator in ``types_validation``, and nothing in the feature +imports it. It uses only the standard library, so it adds no dependency +to a package whose sole runtime dependency is ``opentelemetry-api``. + +Three things this layer does *not* do, on purpose: + +- **It does not verify content.** Verification lives at the accessor boundary in + ``skills_core`` so that it applies to every store equally, including a + customer's own. +- **It does not work around a missing ``contentHash``.** A hashless object is + held verbatim and *withheld* by verification with ``missing_content_hash``. + This module's job is to make that outcome loud — see ``StoreDiagnostics``. +- **It does not evaluate anything.** Flag and segment objects that share the + connection are skipped and counted, nothing more. + +The design rationale — why ``objectVersion`` is not ``version``, why changes +commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery +transport*. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from typing import Any + +from .types_validation import is_valid_skill_version + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# The wire contract +# --------------------------------------------------------------------------- + +FDV2_OBJECT_KIND = "inline-resource" +""" +The FDv2 ``kind`` skills are delivered under. Together with +``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value +``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +""" + +FDV2_OBJECT_CATEGORY = "skill" +"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" + +SDK_DATA_MODEL_VERSION = 1 +""" +The ``mv`` request parameter. The one request parameter whose value could not be +confirmed against a live server, so treat the default as provisional and +override it through the store's ``data_model_version`` if needed. +""" + +_EVENT_SERVER_INTENT = "server-intent" +_EVENT_PUT_OBJECT = "put-object" +_EVENT_DELETE_OBJECT = "delete-object" +_EVENT_PAYLOAD_TRANSFERRED = "payload-transferred" +_EVENT_HEARTBEAT = "heart-beat" +_EVENT_GOODBYE = "goodbye" +_EVENT_ERROR = "error" + +_INTENT_TRANSFER_FULL = "xfer-full" +_INTENT_TRANSFER_CHANGES = "xfer-changes" +_INTENT_TRANSFER_NONE = "none" + +_ENVELOPE_FIELDS = ("contentType", "content", "contentHash", "name", "description") +""" +The skill object envelope's fields, copied through verbatim. Nothing is coerced +or defaulted: a transport that filled in a missing field would be forging the +very thing verification exists to check. +""" + +_MOBILE_KEY_PREFIX = "mob-" +_SERVER_KEY_PREFIX = "sdk-" +_CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") +"""A client-side environment ID: bare lowercase hex. Server-side and mobile keys +both carry a prefix, so this shape is unambiguous rather than heuristic.""" + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +def _require_server_side_credential(sdk_key: str) -> None: + """ + Refuses a mobile key or a client-side environment ID. + + Skill content is customer-confidential, and payload assignment is shared + across credential types, so a client-side credential may well *succeed* + against these endpoints. Raises rather than logs: a store built on the wrong + credential should not exist. + """ + if not isinstance(sdk_key, str) or not sdk_key.strip(): + raise ValueError( + "FDv2SkillStore requires a LaunchDarkly server-side SDK key " + "(sdk-...); none was given." + ) + key = sdk_key.strip() + if key.startswith(_MOBILE_KEY_PREFIX): + raise ValueError( + "FDv2SkillStore was given a mobile key (mob-...). Agent Skills are a " + "server-side feature: skill content is customer-confidential and is " + "never delivered to a mobile or client-side process. Use the " + "environment's server-side SDK key (sdk-...)." + ) + if _CLIENT_SIDE_ID.match(key): + raise ValueError( + "FDv2SkillStore was given what looks like a client-side environment " + "ID. Agent Skills are a server-side feature: skill content is " + "customer-confidential and is never delivered to a client-side " + "process. Use the environment's server-side SDK key (sdk-...)." + ) + if not key.startswith(_SERVER_KEY_PREFIX): + # Not rejected: private instances and test doubles issue credentials + # without the public prefix. Only the two unambiguous shapes above are. + logger.warning( + "The credential given to FDv2SkillStore does not look like a " + "LaunchDarkly server-side SDK key (sdk-...). Skills are delivered " + "only to server-side credentials; if this is a client-side or mobile " + "credential the connection will be rejected or will deliver nothing." + ) + + +# --------------------------------------------------------------------------- +# Diagnostics +# --------------------------------------------------------------------------- + + +@dataclass +class StoreDiagnostics: + """ + What the transport has seen. Read-only from a caller's perspective. + + Not part of the ``SkillStore`` interface. It exists because "this environment + has no skills" and "every skill was withheld" are easy to mistake for each + other, and a counter is easier to assert on than a log line. + """ + + payloads_transferred: int = 0 + """Completed ``payload-transferred`` commits since the store started.""" + skill_objects_received: int = 0 + """``put-object`` events identified as skills, across all payloads.""" + objects_ignored: int = 0 + """Objects skipped because they were not skills: flags, segments, and any + future kind. Skipping is the contract, not a failure.""" + objects_revoked: int = 0 + """``delete-object`` events applied to skills.""" + hashless_objects: int = 0 + """ + Skill objects whose envelope carried no ``contentHash``. + + **Nonzero means skills are being withheld**: verification withholds every one + of these with ``missing_content_hash``. + """ + connection_failures: int = 0 + """Recoverable transport failures since the last successful transfer.""" + last_error: str | None = None + """The most recent transport error, if any. Human-readable; do not parse.""" + + +# --------------------------------------------------------------------------- +# Deserialisation — where objectVersion is not version +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Tombstone: + """A ``delete-object`` narrowed to the identity it revokes.""" + + key: str + object_version: int | None + + +def _is_skill_event(data: Any) -> bool: + """ + Whether one ``put-object`` / ``delete-object`` payload is a skill. + + Both halves are required: ``inline-resource`` may carry other categories, + and flags and segments omit ``category`` entirely. Every other kind is + ignored, not rejected, because flag and segment objects share the connection + and erroring on them would turn a normal payload into a reconnect loop. + """ + if not isinstance(data, dict): + return False + return ( + data.get("kind") == FDV2_OBJECT_KIND + and data.get("category") == FDV2_OBJECT_CATEGORY + ) + + +def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: + """ + Translates one FDv2 skill ``put-object`` into the raw object shape the + ``SkillStore`` interface defines. + + **The one translation this adapter must get right:** + + wire ``objectVersion`` → stored ``version`` (the skill's own version) + wire ``version`` → dropped (the *payload* version) + + ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` + moves whenever anything in the environment moves. Confusing them fails + silently: the object verifies and the caller gets content under a version + number that means nothing. + + Returns ``None`` only when ``key`` is not a string, since a keyless object + has no identity to store it under. Every other defect is carried through + verbatim so that verification withholds it with a reason code rather than + the transport dropping it into indistinguishable absence. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill put-object carried no string 'key' and could not be " + "stored under any identity; it was dropped." + ) + return None + + raw: dict[str, Any] = {"key": key} + + # A membership test rather than a `.get` default, so an explicitly-null + # objectVersion stays null and reaches verification as `invalid_version`. + if "objectVersion" in data: + raw["version"] = data["objectVersion"] + + envelope = data.get("object") + if isinstance(envelope, dict): + for wire_field in _ENVELOPE_FIELDS: + if wire_field in envelope: + raw[wire_field] = envelope[wire_field] + return raw + + +def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: + """ + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with + the same ``objectVersion`` translation as a put. + + An ``object_version`` of ``None`` means the delete named no usable version + and is read as "revoke every version of this key". That is the safe + direction: the alternative is continuing to serve content LaunchDarkly has + withdrawn. + """ + key = data.get("key") + if not isinstance(key, str) or not key: + logger.warning( + "An FDv2 skill delete-object carried no string 'key'; it was ignored." + ) + return None + object_version = data.get("objectVersion") + return _Tombstone( + key=key, + object_version=object_version + if is_valid_skill_version(object_version) + else None, + ) + + +# --------------------------------------------------------------------------- +# The held object set +# --------------------------------------------------------------------------- + + +class _SkillObjectSet: + """ + Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + + Lookup semantics are identical to ``InMemorySkillStore``'s, down to the + fall-through to a version-less entry, so that the store a caller configures + cannot change how a pinned reference resolves; ``TestInterfaceParity`` + asserts it. Reimplemented rather than inherited because the transport needs + ``delete`` and the atomic ``replace_with`` a full transfer requires. + + An object too malformed to carry a usable version is still held, under its + key alone, so verification withholds it with a signal rather than the + transport dropping it. + """ + + def __init__(self) -> None: + self._versions: dict[str, dict[int, dict[str, Any]]] = {} + self._loose: dict[str, dict[str, Any]] = {} + + def put(self, raw: dict[str, Any]) -> None: + key = raw["key"] + version = raw.get("version") + if is_valid_skill_version(version): + self._versions.setdefault(key, {})[version] = raw + else: + self._loose[key] = raw + + def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: + """Removes what *tombstone* revokes; returns the raw objects that went away.""" + removed: list[dict[str, Any]] = [] + if tombstone.object_version is None: + held = self._versions.pop(tombstone.key, {}) + removed.extend(held.values()) + loose = self._loose.pop(tombstone.key, None) + if loose is not None: + removed.append(loose) + return removed + + held = self._versions.get(tombstone.key, {}) + gone = held.pop(tombstone.object_version, None) + if gone is not None: + removed.append(gone) + if not held: + self._versions.pop(tombstone.key, None) + return removed + + def get(self, key: str, version: int | None) -> dict[str, Any] | None: + held = self._versions.get(key, {}) + if version is not None: + # Fall through to the version-less entry so a malformed object + # reaches verification rather than reading as simply absent. + return held.get(version) or self._loose.get(key) + if held: + return held[max(held)] + return self._loose.get(key) + + def snapshot(self) -> dict[str, dict[str, Any]]: + """One entry per ``(key, version)``, under keys opaque to the SDK.""" + out: dict[str, dict[str, Any]] = { + f"{key}:{version}": raw + for key, versions in self._versions.items() + for version, raw in versions.items() + } + out.update(self._loose) + return out + + def all_raw(self) -> list[dict[str, Any]]: + return list(self.snapshot().values()) + + def replace_with(self, other: _SkillObjectSet) -> None: + """Adopts *other*'s contents wholesale — how a full transfer commits.""" + self._versions = other._versions + self._loose = other._loose + + def copy(self) -> _SkillObjectSet: + clone = _SkillObjectSet() + clone._versions = {key: dict(v) for key, v in self._versions.items()} + clone._loose = dict(self._loose) + return clone + + def __len__(self) -> int: + return sum(len(v) for v in self._versions.values()) + len(self._loose) + + +# --------------------------------------------------------------------------- +# The protocol state machine — pure, no I/O +# --------------------------------------------------------------------------- + + +@dataclass +class _TransferOutcome: + """What one event did. Aggregated by the caller; nothing here does I/O.""" + + committed: bool = False + changes: list[dict[str, Any]] = field(default_factory=list) + basis: str | None = None + fatal: str | None = None + disconnect: str | None = None + + +class _ProtocolReader: + """ + Applies FDv2 events to an object set. Pure — no sockets, no threads, no clock — + so every wire case is testable without a server. + + **Changes are buffered and committed at ``payload-transferred``.** A payload + version is the unit of consistency: applying half of one would publish a + state the server never described, and on a full transfer would briefly empty + the store. Listeners therefore fire once per commit, not once per object. + """ + + def __init__(self, committed: _SkillObjectSet) -> None: + self._committed = committed + self._intent: str | None = None + self._pending: _SkillObjectSet | None = None + self._changes: list[dict[str, Any]] = [] + self.diagnostics = StoreDiagnostics() + # Identities already reported by ``_warn_hashless``. Per reader, so a + # recreated store reports again and two stores never quieten each other. + # No lock: ``handle`` runs only on its owner's single delivery thread. + self._warned_hashless: set[tuple[str, Any]] = set() + + # -- events ------------------------------------------------------------ + + def handle(self, name: str, data: Any) -> _TransferOutcome: + """Routes one event. Unknown event names are ignored, by contract.""" + if name == _EVENT_SERVER_INTENT: + return self._server_intent(data) + if name == _EVENT_PUT_OBJECT: + return self._put_object(data) + if name == _EVENT_DELETE_OBJECT: + return self._delete_object(data) + if name == _EVENT_PAYLOAD_TRANSFERRED: + return self._payload_transferred(data) + if name == _EVENT_ERROR: + return self._error(data) + if name == _EVENT_GOODBYE: + return self._goodbye(data) + if name == _EVENT_HEARTBEAT: + return _TransferOutcome() + logger.debug("Ignoring unknown FDv2 event '%s'", name) + return _TransferOutcome() + + def _server_intent(self, data: Any) -> _TransferOutcome: + payloads = data.get("payloads") if isinstance(data, dict) else None + if not isinstance(payloads, list) or not payloads: + return _TransferOutcome( + disconnect="server-intent carried no payload description" + ) + first = payloads[0] + intent = first.get("intentCode") if isinstance(first, dict) else None + self._intent = intent + self._changes = [] + if intent == _INTENT_TRANSFER_FULL: + # Built alongside the live set rather than in place, so an + # interrupted transfer leaves last known good intact. + self._pending = _SkillObjectSet() + elif intent == _INTENT_TRANSFER_CHANGES: + self._pending = self._committed.copy() + else: + if intent != _INTENT_TRANSFER_NONE: + logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) + self._pending = None + return _TransferOutcome() + + def _target_for(self, data: Any) -> _SkillObjectSet | None: + """ + The pending set a skill object event applies to, or ``None`` when the + event is not a skill or the current intent carries no objects. + + An object arriving with no ``server-intent`` at all is treated as a + delta against what is held rather than dropped. + """ + if not _is_skill_event(data): + self.diagnostics.objects_ignored += 1 + return None + if self._pending is None: + if self._intent is None: + self._intent = _INTENT_TRANSFER_CHANGES + if self._intent not in (_INTENT_TRANSFER_FULL, _INTENT_TRANSFER_CHANGES): + return None + self._pending = self._committed.copy() + return self._pending + + def _put_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + raw = _store_object_from_put(data) + if raw is None: + return _TransferOutcome() + target.put(raw) + self._changes.append(raw) + self.diagnostics.skill_objects_received += 1 + if not isinstance(raw.get("contentHash"), str): + self.diagnostics.hashless_objects += 1 + self._warn_hashless(raw) + return _TransferOutcome() + + def _delete_object(self, data: Any) -> _TransferOutcome: + target = self._target_for(data) + if target is None: + return _TransferOutcome() + tombstone = _tombstone_from_delete(data) + if tombstone is None: + return _TransferOutcome() + target.delete(tombstone) + self.diagnostics.objects_revoked += 1 + # A tombstone carries identity and no content, so a listener that reads + # content must check for ``content`` rather than assume it. + self._changes.append( + {"key": tombstone.key, "version": tombstone.object_version} + ) + return _TransferOutcome() + + def _payload_transferred(self, data: Any) -> _TransferOutcome: + state = data.get("state") if isinstance(data, dict) else None + version = data.get("version") if isinstance(data, dict) else None + if self._pending is not None: + self._committed.replace_with(self._pending) + _warn_if_nothing_can_verify(self._committed) + self._pending = None + self._intent = None + changes = self._changes + self._changes = [] + self.diagnostics.payloads_transferred += 1 + logger.debug( + "FDv2 payload transferred: payload version %s, %d skill object(s) held", + version, + len(self._committed), + ) + return _TransferOutcome( + committed=True, + changes=changes, + basis=state if isinstance(state, str) and state else None, + ) + + def _abandon_in_flight(self) -> None: + """Drops the in-flight payload and keeps what is committed.""" + self._pending = None + self._intent = None + self._changes = [] + + def _error(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + self._abandon_in_flight() + return _TransferOutcome(disconnect=f"server sent error: {reason}") + + def _goodbye(self, data: Any) -> _TransferOutcome: + reason = data.get("reason") if isinstance(data, dict) else None + catastrophe = bool(data.get("catastrophe")) if isinstance(data, dict) else False + silent = bool(data.get("silent")) if isinstance(data, dict) else False + self._abandon_in_flight() + if not silent: + logger.info("FDv2 connection closing: %s", reason) + if catastrophe: + return _TransferOutcome( + fatal=f"server sent a catastrophic goodbye: {reason}" + ) + return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + + # -- diagnostics --------------------------------------------------------- + + def _warn_hashless(self, raw: dict[str, Any]) -> None: + """ + One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. + + ERROR rather than WARN because an empty accessor result is otherwise + indistinguishable from an environment that has no skills. + """ + identity = (raw["key"], raw.get("version")) + if identity in self._warned_hashless: + return + self._warned_hashless.add(identity) + logger.error( + "Skill '%s' version %s arrived without a contentHash and will be withheld. %s", + raw["key"], + raw.get("version"), + _HASHLESS_ADVICE, + extra={"ld_skill_key": raw["key"], "ld_skill_version": raw.get("version")}, + ) + + +_HASHLESS_ADVICE = ( + "The delivered skill object carries no 'contentHash', so integrity " + "verification withholds it with reason_code 'missing_content_hash' and its " + "content will not resolve. The SDK cannot work around this: verification " + "hashes the delivered bytes and compares them against the envelope's " + "'contentHash', and there is nothing to compare against. 'contentHash' is a " + "sha256 over the verbatim UTF-8 content. Contact LaunchDarkly support if " + "skills in your environment arrive without one." +) + + +def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: + """ + One ERROR per committed payload in which *nothing* held can possibly verify. + + ``log_withholding_summary`` reports the same condition at the accessor + boundary, but only once a caller asks. This fires at delivery time, so it is + visible in a process that boots, materializes nothing, and exits. + """ + held = committed.all_raw() + if not held: + return + hashless = [raw for raw in held if not isinstance(raw.get("contentHash"), str)] + if len(hashless) != len(held): + return + logger.error( + "All %d skill object(s) in the delivered payload arrived without a " + "contentHash. No skill content will resolve from this store. %s", + len(held), + _HASHLESS_ADVICE, + ) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py new file mode 100644 index 00000000..ff5720bc --- /dev/null +++ b/packages/client/tests/test_skills_fdv2.py @@ -0,0 +1,605 @@ +""" +Tests for the FDv2 skill delivery protocol. + +Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, +revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +against ``_ProtocolReader``, which has no I/O, so each case reads as the contract +it is rather than as a server script. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, ClassVar + +import pytest + +from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND +from launchdarkly_ai_server.skills_fdv2 import ( + FDV2_OBJECT_CATEGORY, + FDV2_OBJECT_KIND, + _is_skill_event, + _ProtocolReader, + _SkillObjectSet, + _store_object_from_put, + _tombstone_from_delete, +) + +pytestmark = pytest.mark.usefixtures("reset_skill_state") + +SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" + + +def _hash(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Wire builders — one place that knows the shape, so a contract change is one edit +# --------------------------------------------------------------------------- + + +def put_skill( + key: str = "pdf-extraction", + *, + object_version: Any = 3, + payload_version: int = 42, + content: str = SKILL_BODY, + content_hash: Any = None, + omit_hash: bool = False, + name: str = "PDF Extraction", +) -> dict[str, Any]: + """One skill ``put-object`` event's data, in the shape the wire delivers it.""" + envelope: dict[str, Any] = { + "contentType": "text/markdown", + "content": content, + "name": name, + "description": "Extracts text", + } + if not omit_hash: + envelope["contentHash"] = ( + content_hash if content_hash is not None else _hash(content) + ) + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + "object": envelope, + } + + +def delete_skill( + key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 +) -> dict[str, Any]: + return { + "key": key, + "kind": FDV2_OBJECT_KIND, + "category": FDV2_OBJECT_CATEGORY, + "objectVersion": object_version, + "version": payload_version, + } + + +def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: + """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + return { + "key": key, + "kind": "flag", + "version": version, + "object": { + "key": key, + "version": version, + "on": True, + "variations": [True, False], + }, + } + + +def put_segment(key: str = "beta-users", version: int = 4) -> dict[str, Any]: + return { + "key": key, + "kind": "segment", + "version": version, + "object": {"key": key, "version": version, "included": []}, + } + + +def server_intent( + code: str = "xfer-full", payload_id: str = "agent-skill" +) -> dict[str, Any]: + return { + "payloads": [ + {"id": payload_id, "target": 1, "intentCode": code, "reason": "test"} + ] + } + + +def transferred(state: str = "basis-1", version: int = 42) -> dict[str, Any]: + return {"state": state, "version": version} + + +def events(*pairs: tuple[str, Any]) -> list[dict[str, Any]]: + return [{"event": name, "data": data} for name, data in pairs] + + +def full_payload( + *object_events: tuple[str, Any], state: str = "basis-1" +) -> list[dict[str, Any]]: + return events( + ("server-intent", server_intent("xfer-full")), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +# --------------------------------------------------------------------------- +# Identifying skill objects, and ignoring everything else +# --------------------------------------------------------------------------- + + +class TestObjectIdentification: + def test_kind_and_category_together_identify_a_skill(self) -> None: + assert _is_skill_event(put_skill()) is True + + def test_a_flag_is_not_a_skill(self) -> None: + assert _is_skill_event(put_flag()) is False + + def test_a_segment_is_not_a_skill(self) -> None: + assert _is_skill_event(put_segment()) is False + + def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: + """``inline-resource`` is a broad kind, so the category is required too.""" + other = put_skill() + other["category"] = "prompt-template" + assert _is_skill_event(other) is False + + def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + other = put_skill() + other["kind"] = "some-future-kind" + assert _is_skill_event(other) is False + + def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: + """Flags and segments omit ``category`` entirely — the documented shape.""" + assert "category" not in put_flag() + assert "objectVersion" not in put_flag() + + @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) + def test_non_dict_events_are_not_skills(self, value: Any) -> None: + assert _is_skill_event(value) is False + + +# --------------------------------------------------------------------------- +# objectVersion is not version +# --------------------------------------------------------------------------- + + +class TestVersionTranslation: + def test_object_version_becomes_the_seam_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] == 3 + + def test_the_payload_version_never_reaches_the_seam(self) -> None: + """ + The failure this asserts against is silent: a store that read ``version`` + would serve verifiable content under a version number that means nothing, + and every pinned reference would resolve to the wrong thing with no error. + """ + raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) + assert raw is not None + assert raw["version"] != 42 + assert 42 not in raw.values() + + def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( + self, + ) -> None: + raw = _store_object_from_put(put_skill(object_version=99, payload_version=1)) + assert raw is not None + assert raw["version"] == 99 + + def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: + wire = put_skill() + del wire["objectVersion"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "version" not in raw + + def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: + """Carried, not invented: verification reports ``invalid_version``.""" + raw = _store_object_from_put(put_skill(object_version=None)) + assert raw is not None + assert raw["version"] is None + + def test_a_delete_translates_object_version_too(self) -> None: + tombstone = _tombstone_from_delete( + delete_skill(object_version=3, payload_version=43) + ) + assert tombstone is not None + assert tombstone.object_version == 3 + + def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + assert tombstone is not None + assert tombstone.object_version is None + + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: + wire = put_skill() + del wire["key"] + assert _store_object_from_put(wire) is None + + def test_the_envelope_is_copied_verbatim(self) -> None: + raw = _store_object_from_put(put_skill()) + assert raw is not None + assert raw["content"] == SKILL_BODY + assert raw["contentHash"] == _hash(SKILL_BODY) + assert raw["name"] == "PDF Extraction" + assert raw["contentType"] == "text/markdown" + + def test_an_absent_envelope_field_is_absent_rather_than_defaulted(self) -> None: + wire = put_skill() + del wire["object"]["name"] + raw = _store_object_from_put(wire) + assert raw is not None + assert "name" not in raw + + +# --------------------------------------------------------------------------- +# The protocol reader +# --------------------------------------------------------------------------- + + +def drive(reader: _ProtocolReader, payload_events: list[dict[str, Any]]) -> list[Any]: + return [reader.handle(e["event"], e.get("data")) for e in payload_events] + + +class TestProtocolReader: + def test_a_full_transfer_commits_at_payload_transferred(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive(reader, full_payload(("put-object", put_skill()))) + assert len(held) == 1 + assert outcomes[-1].committed is True + assert outcomes[-1].basis == "basis-1" + + def test_nothing_is_visible_before_payload_transferred(self) -> None: + """A payload version is the unit of consistency; half of one is not a state.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill()), + ), + ) + assert len(held) == 0 + + def test_an_interrupted_full_transfer_leaves_last_known_good_intact(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + assert held.get("pdf-extraction", None) is not None + + # A second full transfer starts and never completes. + drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ), + ) + still_held = held.get("pdf-extraction", None) + assert still_held is not None + assert still_held["version"] == 1 + + def test_a_full_transfer_replaces_rather_than_merges(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, full_payload(("put-object", put_skill("second")), state="basis-2") + ) + assert held.get("first", None) is None + assert held.get("second", None) is not None + + def test_a_change_transfer_applies_deltas_over_what_is_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill("first")))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("first", None) is not None + assert held.get("second", None) is not None + + def test_a_delete_object_revokes_the_skill(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=3)))) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", None) is None + assert reader.diagnostics.objects_revoked == 1 + + def test_a_delete_notifies_with_a_tombstone_carrying_no_content(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ), + ) + (change,) = outcomes[-1].changes + assert change == {"key": "pdf-extraction", "version": 3} + assert "content" not in change + + def test_a_delete_for_one_version_leaves_the_other_held(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + full_payload( + ("put-object", put_skill(object_version=2)), + ("put-object", put_skill(object_version=3)), + ), + ) + drive( + reader, + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill(object_version=3)), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert held.get("pdf-extraction", 2) is not None + assert held.get("pdf-extraction", None)["version"] == 2 + + def test_flag_and_segment_objects_are_skipped_cleanly(self) -> None: + """ + The mixed payload is the normal case, not an edge one: an environment's + assignment carries its flag payload alongside its agent-skill payload. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcomes = drive( + reader, + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_segment("beta-users")), + ("put-object", put_flag("flag-b")), + ("delete-object", put_flag("flag-c")), + ), + ) + assert len(held) == 1 + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.objects_ignored == 4 + assert reader.diagnostics.skill_objects_received == 1 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_kind_is_ignored_rather_than_fatal(self) -> None: + """ + Erroring on an unrecognised kind would turn a normal payload into a + permanent reconnect loop — a flag-delivery outage caused by a skills + rollout. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + exotic = { + "key": "x", + "kind": "quantum-widget", + "version": 1, + "object": {"a": 1}, + } + outcomes = drive(reader, full_payload(("put-object", exotic))) + assert len(held) == 0 + assert all(o.fatal is None and o.disconnect is None for o in outcomes) + + def test_an_unknown_event_name_is_ignored(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + outcome = reader.handle("some-future-event", {"anything": True}) + assert outcome.fatal is None + assert outcome.disconnect is None + + def test_a_heartbeat_does_nothing(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("heart-beat", None) + assert outcome == type(outcome)() + + def test_an_error_event_abandons_the_in_flight_payload(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill(object_version=1)))) + outcomes = drive( + reader, + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=2)), + ( + "error", + {"payloadId": "agent-skill", "reason": "backend unavailable"}, + ), + ), + ) + assert outcomes[-1].disconnect is not None + assert held.get("pdf-extraction", None)["version"] == 1 + + def test_a_goodbye_asks_for_a_reconnect(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("goodbye", {"reason": "rebalancing", "silent": False}) + assert outcome.disconnect is not None + assert outcome.fatal is None + + def test_a_catastrophic_goodbye_is_fatal(self) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle( + "goodbye", {"reason": "no", "silent": False, "catastrophe": True} + ) + assert outcome.fatal is not None + + def test_transfer_none_holds_everything_and_commits(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, full_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", server_intent("none")), + ("payload-transferred", transferred("basis-2")), + ), + ) + assert len(held) == 1 + + def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ("put-object", put_skill()), + ("payload-transferred", transferred("basis-1")), + ), + ) + assert len(held) == 1 + + +# --------------------------------------------------------------------------- +# Interface parity with InMemorySkillStore +# --------------------------------------------------------------------------- + + +class TestInterfaceParity: + """ + The two stores must resolve identically. ``_SkillObjectSet`` reimplements the + lookup rather than inheriting it — see its docstring for why — so this is the + test that stops the two from drifting. + """ + + RAWS: ClassVar[list[dict[str, Any]]] = [ + {"key": "a", "version": 1, "content": "x", "contentHash": _hash("x")}, + {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, + {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, + {"key": "malformed", "version": "not-a-version", "content": "q"}, + ] + + def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: + memory = InMemorySkillStore() + objects = _SkillObjectSet() + for raw in self.RAWS: + memory.put(dict(raw)) + objects.put(dict(raw)) + return memory, objects + + @pytest.mark.parametrize( + "key,version", + [ + ("a", None), + ("a", 1), + ("a", 4), + ("a", 9), + ("b", 2), + ("b", None), + ("missing", None), + ("missing", 1), + ("malformed", None), + ("malformed", 7), + ], + ) + def test_get_agrees(self, key: str, version: int | None) -> None: + memory, objects = self._both() + assert memory.get_object(SKILL_OBJECT_KIND, key, version) == objects.get( + key, version + ) + + def test_snapshot_agrees(self) -> None: + memory, objects = self._both() + assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() + + +# --------------------------------------------------------------------------- +# The contentHash gap +# --------------------------------------------------------------------------- + + +def _per_object_hashless_errors(caplog: Any) -> list[Any]: + """The per-object ERROR, as distinct from the whole-payload summary.""" + return [ + r + for r in caplog.records + if r.levelname == "ERROR" + and "arrived without a contentHash" in r.getMessage() + and "No skill content will resolve" not in r.getMessage() + ] + + +class TestMissingContentHash: + """ + A skill delivered without a ``contentHash``, asserted as behaviour. + + An envelope with no ``contentHash`` must produce a *withheld* skill with the + ``missing_content_hash`` reason — loudly, diagnosably, and without a crash. + There is deliberately no fallback that skips verification: a hash the SDK + computed from the content it was handed would certify the content against + itself and verify nothing. + """ + + def test_a_redelivered_hashless_object_logs_once_per_store( + self, caplog: Any + ) -> None: + """Re-delivering the same ``(key, version)`` to one store must not + multiply the ERROR: a polling store sees every object on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(reader, payload) + drive(reader, payload) + assert len(_per_object_hashless_errors(caplog)) == 1 + + def test_a_recreated_store_reports_the_same_hashless_object_again( + self, caplog: Any + ) -> None: + """ + The dedupe belongs to the store, not the process. A host that rebuilds + its store (reconnect wrapper, config reload, credential rotation) must + get the ERROR again, since it is the loudest signal that a deployment is + broken rather than empty by design. + """ + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(_ProtocolReader(_SkillObjectSet()), payload) + first = len(_per_object_hashless_errors(caplog)) + drive(_ProtocolReader(_SkillObjectSet()), payload) + assert first == 1 + assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: + """Two stores in one process (say, two environments) each report.""" + one = _ProtocolReader(_SkillObjectSet()) + two = _ProtocolReader(_SkillObjectSet()) + payload = full_payload(("put-object", put_skill(omit_hash=True))) + with caplog.at_level("ERROR"): + drive(one, payload) + drive(two, payload) + # And each still dedupes its own re-deliveries. + drive(one, payload) + drive(two, payload) + assert len(_per_object_hashless_errors(caplog)) == 2 From e2b54fd990ff94a415e13f9c6b6bdec4a7794410 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 13:43:52 -0400 Subject: [PATCH 17/43] feat(client): an unheld version pin is a miss, not an integrity failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``InMemorySkillStore.get_object`` consulted the version-less entry on any pinned miss, while the unpinned path consulted it only when nothing well-formed was filed under the key. A key holding both well-formed versions and one malformed object therefore answered a pin for an undelivered version with the malformed object, and verification recorded an integrity failure — an alert pointed at a skill whose integrity was never in question — where the honest answer is that the version is not held. Both paths now follow the one rule: the version-less entry answers only when nothing well-formed is filed under the key, which is the case it exists for. A malformed object that is all the store holds still reaches verification and is still withheld with a signal, so tampering cannot read as a skill that was never delivered. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills.py | 17 ++++---- packages/client/tests/test_skills.py | 39 +++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index eb3946c2..36ae7dee 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -131,14 +131,17 @@ def get_object( if kind != SKILL_OBJECT_KIND: return None held = self._versions.get(key, {}) + if not held: + # Nothing well-formed is filed under this key, so the version-less + # entry is all there is: serve it, and let verification withhold it + # with a signal rather than have it read as simply absent. A pin that + # misses while well-formed versions do exist is a plain miss, and + # answering it with a leftover malformed object would record an + # integrity failure for a skill whose integrity is not in question. + return self._loose.get(key) if version is not None: - # Fall through to the version-less entry when the pin does not match - # anything well-formed, so a malformed object reaches verification and - # is withheld with a signal rather than reading as simply absent. - return held.get(version) or self._loose.get(key) - if held: - return held[max(held)] - return self._loose.get(key) + return held.get(version) + return held[max(held)] def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: """ diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 203eeb4f..6b9dfe34 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -378,6 +378,23 @@ def test_get_object_unknown_version_falls_back_to_a_malformed_object( assert s.get_object("skill", "a", 2) == malformed assert s.get_object("skill", "a") == malformed + def test_get_object_unknown_version_beside_well_formed_ones_is_absent( + self, make_raw_skill: Any + ) -> None: + """A pin miss is a miss, not an integrity failure. + + The fall-back above applies only when nothing well-formed is filed under + the key. Once well-formed versions are held, a leftover malformed object + must not answer for a version that was never delivered: verification + would withhold it and record an integrity failure against a skill whose + integrity is not in question. + """ + s = InMemorySkillStore() + s.put(make_raw_skill(key="a", version=1, content="one\n")) + s.put(make_raw_skill(key="a", version=2, content="two\n")) + s.put(make_raw_skill(key="a", version="two")) + assert s.get_object("skill", "a", 5) is None + def test_all_objects_unknown_kind_is_empty(self, make_raw_skill: Any) -> None: s = InMemorySkillStore() s.put(make_raw_skill(key="a")) @@ -769,6 +786,28 @@ async def test_pin_to_a_version_the_store_does_not_hold_returns_none( await self._two_versions(store, make_raw_skill) assert await get_skill("a", version=9) is None + async def test_pin_miss_beside_a_malformed_object_records_no_failure( + self, + store: Any, + make_raw_skill: Any, + recording_emitter: Any, + ) -> None: + """An undelivered version must not raise an integrity alarm. + + A malformed object is filed under its key alone, and a pinned lookup + serves it when that is all the store holds — so verification withholds it + with a signal rather than letting tampering read as a skill that was + never delivered. Once well-formed versions are held, that reasoning no + longer applies: the pin is simply not there, and reporting an integrity + failure would point an alert at the wrong skill. + """ + skills_module._set_emitter_for_testing(recording_emitter) + await self._two_versions(store, make_raw_skill) + store.put(make_raw_skill(key="a", version="two")) + + assert await get_skill("a", version=9) is None + assert recording_emitter.signals(INTEGRITY_SIGNAL) == [] + async def test_all_skills_returns_one_entry_per_key_at_the_newest_version( self, store: Any, make_raw_skill: Any ) -> None: From 5817d89136b74dd2dbdd36b4cafdf71161c47275 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:16:17 -0400 Subject: [PATCH 18/43] fix(client): a broken store answer is not an absent skill Two untrusted-store answers were read as data rather than as failures. ``list_raw_objects`` collapsed a non-mapping listing to ``{}`` with no error, so a store that served nothing usable was indistinguishable from one holding no skills. ``resolve_from_store`` read identity off the object without checking it against the key that was asked for, so an answer served under a different key came back under the caller's key while carrying its own. Both are now withheld and reported, alongside the version check that already guarded the same way. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_core.py | 28 ++++++++++- packages/client/tests/test_skills.py | 50 +++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 1e517e4e..a3894335 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -621,13 +621,27 @@ def list_raw_objects( worded identically. Letting the exception out instead would make each of them re-derive the log line and the message, which is the drift this module exists to prevent. + + An answer that is not a mapping is a broken store, on the same footing as + one that raised — **not** an empty one. Collapsing it to ``{}`` would make a + store that served nothing usable indistinguishable from a store that holds + no skills, which reads downstream as "every skill was revoked". """ try: objects = store.all_objects(SKILL_OBJECT_KIND) except Exception as exc: logger.error("Skill store raised while listing skills", exc_info=True) return {}, store_raised(exc) - return (objects if isinstance(objects, dict) else {}), None + if not isinstance(objects, dict): + logger.error( + "Skill store listed skills as %s rather than an object", + type(objects).__name__, + ) + return {}, ( + f"the skill store listed skills as {type(objects).__name__} " + "rather than an object" + ) + return objects, None def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: @@ -696,7 +710,10 @@ def resolve_from_store( versions of one key and only it can pick between them; ``None`` asks for the newest. The equality check afterwards is kept as a **defense**, not as the selection mechanism: the store is untrusted, so an answer that is not the - version that was asked for is withheld rather than returned. + version that was asked for is withheld rather than returned. The key is + checked the same way and for the same reason: identity is read off the + object itself, so an answer served under a different key would otherwise be + returned under the caller's key while carrying its own. """ try: raw = store.get_object(SKILL_OBJECT_KIND, key, wanted_version) @@ -714,6 +731,13 @@ def resolve_from_store( return Resolution( error=f"skill '{key}' failed integrity verification and was withheld" ) + if skill.key != key: + return Resolution( + error=( + f"skill '{key}' is not available: the store answered under " + f"key '{skill.key}'" + ) + ) if wanted_version is not None and skill.version != wanted_version: return Resolution( error=( diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 6b9dfe34..0b0cadef 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -27,6 +27,7 @@ shutdown, skill_refs, ) +from launchdarkly_ai_server.skills_core import list_raw_objects, require_store SKILL_BODY = "---\nname: Test Skill\n---\nDo the thing.\n" @@ -631,6 +632,28 @@ async def test_other_version_returns_none( async def test_missing_key_returns_none(self, store: InMemorySkillStore) -> None: assert await get_skill("nope") is None + async def test_a_store_answering_under_a_different_key_is_withheld( + self, make_raw_skill: Any + ) -> None: + """The key needs the same post-fetch defense the version already has. + + Identity is read off the object itself, and the store is untrusted. An + answer served under a different key would otherwise be handed back + under the key the caller asked for while carrying its own. + """ + + class _AliasingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key="other-key") + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_AliasingStore()) + assert await get_skill("asked-for") is None + async def test_multibyte_content_verifies( self, store: InMemorySkillStore, make_raw_skill: Any ) -> None: @@ -724,6 +747,33 @@ async def test_omits_skills_that_fail_verification( assert {s.key for s in result} == {"good"} assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 + async def test_a_non_mapping_listing_is_reported_as_a_broken_store(self) -> None: + """A listing that is not a mapping is a broken store, not an empty one. + + ``all_skills`` has no way to report the difference, so it returns an + empty list either way — but the reason has to reach the caller that + does act on it. Collapsing the answer to "no skills" reads downstream + as "every skill was revoked". + """ + + class _BrokenListingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + skills_module._set_store(_BrokenListingStore()) + + assert await all_skills() == [] + + objects, error = list_raw_objects(require_store()) + assert objects == {} + assert error is not None + assert "rather than an object" in error + class TestVersionPinning: """ From 29ad3feda3c50808d0673e4b21171cdf2b250230 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:17:57 -0400 Subject: [PATCH 19/43] fix(client): attach the skill watcher's listener before its first reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit watch_skills awaited write_skills and only then constructed SkillWatcher, which is where the store listener attaches. The reconcile snapshots the store as its first step and then spends the rest of its time on the filesystem, so every write, fsync, prune and manifest rewrite in that first pass ran with nothing listening. A change delivered in that window was never seen, and since nothing re-reconciles on a timer, a revocation that landed there waited for the next unrelated change — on a quiet root, the next restart. Exactly the gap watch_skills exists to close. The watcher now attaches its listener before the initial reconcile and starts its worker after. notify only sets an event, so a change arriving mid-reconcile is recorded and picked up by the worker's first pass, while holding the thread back keeps write_skills's one-root-one-reconcile contract: the worker cannot race the caller's own reconcile over the same manifest. A reconcile that raises detaches the listener on the way out, since the caller is handed an exception rather than a watcher to close. Co-Authored-By: Claude Opus 5 --- .../launchdarkly_ai_server/skills_watch.py | 51 ++++++++++++--- packages/client/tests/test_skills_watch.py | 62 +++++++++++++++++++ 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..94b31d46 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -94,13 +94,26 @@ def __init__( target=self._run, name="ld-ai-skills-reconcile", daemon=True ) - # Register before starting the worker: ``notify`` only sets an event, so a - # change that lands in between is picked up as soon as the worker runs, - # and a store whose ``add_listener`` raises leaves no thread behind. + # Register before the initial reconcile, and leave the worker unstarted + # until ``start``. ``notify`` only sets an event, so a change that lands + # while that reconcile is still running is recorded rather than lost, and + # the worker cannot reconcile the root while the caller's own reconcile is + # in flight. A store whose ``add_listener`` raises leaves no thread behind. self._store = store self._registered = False store.add_listener(SKILL_OBJECT_KIND, self.notify) self._registered = True + + def _start(self) -> None: + """ + Starts the worker. ``watch_skills`` calls this once, after the initial + reconcile; it is not part of the caller-facing interface. + + Split from construction so registration and reconciling can be ordered + independently: the listener attaches first, so no change is missed, while + the first re-reconcile waits for the initial one to finish, so a root only + ever has one reconcile running at a time. + """ self._thread.start() # -- the listener the store calls ------------------------------------- @@ -272,13 +285,13 @@ async def watch_skills( if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") - # The initial reconcile runs first and on the caller's thread, so its report - # is the caller's to inspect and a bad root raises out of `watch_skills` - # rather than into a worker thread's log. - report = await write_skills( - skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable - ) - + # The watcher attaches its listener before the initial reconcile, not after. + # The reconcile snapshots the store as its first step and then spends the + # rest of its time on the filesystem — a write and an fsync per skill, the + # prune, the manifest rewrite — so a change delivered after that snapshot + # needs something already listening to be seen at all. Nothing re-reconciles + # on a timer, so a revocation that landed unobserved would wait for the next + # unrelated change, which on a quiet root means the next restart. watcher = SkillWatcher( skills, root, @@ -289,4 +302,22 @@ async def watch_skills( debounce=debounce, on_reconcile=on_reconcile, ) + try: + # The initial reconcile runs on the caller's thread, so its report is the + # caller's to inspect and a bad root raises out of `watch_skills` rather + # than into a worker thread's log. + report = await write_skills( + skills, root, prune=prune, timeout=timeout, on_unavailable=on_unavailable + ) + except BaseException: + # The listener is already attached, so a reconcile that raises must not + # leave it on the store: the caller has no watcher to close. + watcher.close() + raise + + # Only now start the worker. A change that arrived during the reconcile has + # already set the wake event, so the worker's first pass picks it up; one that + # arrived before the reconcile's snapshot is already on disk, and the + # redundant pass it triggers converges on the same state. + watcher._start() return report, watcher diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..dada2015 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -92,6 +92,68 @@ async def test_no_store_configured_raises(self, tmp_path: Any) -> None: await watch_skills("*", tmp_path / "s") +# --------------------------------------------------------------------------- +# Changes that land while the initial reconcile is running +# --------------------------------------------------------------------------- + + +class TestChangesDuringTheInitialReconcile: + """The listener attaches before the initial reconcile, so a change delivered + while that reconcile is still running is acted on rather than lost.""" + + async def test_a_revocation_landing_mid_reconcile_is_not_missed( + self, tmp_path: Any, make_raw_skill: Any + ) -> None: + class RevokesAfterSnapshot(InMemorySkillStore): + """Revokes everything the moment the reconcile has taken its + snapshot — where a ``delete-object`` lands when it arrives a fraction + of a second into startup, with nothing after it.""" + + def __init__(self) -> None: + super().__init__() + self.snapshots = 0 + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + objects = super().all_objects(kind) + self.snapshots += 1 + if self.snapshots == 1: + self._versions.clear() + self._loose.clear() + for listener in self._listeners.get(SKILL_OBJECT_KIND, []): + listener({"key": "pdf-extraction"}) + return objects + + store = RevokesAfterSnapshot() + store.put(make_raw_skill(key="pdf-extraction", version=1, content="body")) + await init_client(options={"skillStore": store}, client=object()) + + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + # The initial reconcile wrote what its snapshot held, so the file is + # on disk and the revocation that followed it is the only change + # left to act on. + assert written.read_text() == "body" + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_failed_initial_reconcile_leaves_no_listener_behind( + self, tmp_path: Any + ) -> None: + """Registering first means a reconcile that raises has to detach: the + caller is handed an exception, not a watcher to close.""" + store = InMemorySkillStore() + await init_client(options={"skillStore": store}, client=object()) + not_a_directory = tmp_path / "file" + not_a_directory.write_text("") + + with pytest.raises(ValueError, match="not a directory"): + await watch_skills("*", not_a_directory) + + assert store._listeners.get(SKILL_OBJECT_KIND, []) == [] + + # --------------------------------------------------------------------------- # Closing a watch # --------------------------------------------------------------------------- From 4c6d965b066941fa1d836a5b53ec6ddfe5e513e4 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:20:57 -0400 Subject: [PATCH 20/43] test(client): a broken store answer never deletes managed files Covers the reconcile side of the withheld-answer fix: a listing that is not a mapping leaves every managed file alone rather than reading as a full revocation, and an answer served under a different key writes nothing, is reported against the key that was asked for, and does not reach that other key's file. Each one previously deleted a file and reported a clean run. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 62d01d59..4aa9ed8e 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -532,6 +532,120 @@ def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: assert _actions_by_key(report)["pdf-extraction"].action == "error" assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + async def test_a_non_mapping_listing_never_prunes(self, root: Path) -> None: + """A store that cannot list is not a store holding nothing. + + A listing collapsed to "no skills" is indistinguishable from every + skill having been revoked, and prune would then delete every managed + file and report a clean run. The listing failure has to reach the + prune gate as an incomplete run. + """ + + class NoListingStore: + """Answers the listing with something that is not a mapping.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return None + + def all_objects(self, kind: str) -> Any: + return None + + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + skills_module._set_store(NoListingStore()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert any("rather than an object" in m for m in _error_messages(report)) + # The entry survives, so the next reconcile picks it up. + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + + async def test_an_answer_under_another_key_writes_nothing(self, root: Path) -> None: + """The file is named after the key the object carries, so a store + answering under a different key would write one path and prune another. + + Left unchecked, the run wrote the aliased key, then deleted it in the + same pass because prune keys off the request — and reported ok. The + requested key has to be the one the outcome is reported against. + """ + + class AliasingStore: + """Answers every lookup with an object carrying its own key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(AliasingStore()) + + report = await write_skills(["requested-key"], root) + + assert report.ok is False + assert [a.action for a in report.actions] == ["error"] + # Reported against the key that was asked for, not the one served. + assert _actions_by_key(report)["requested-key"].action == "error" + assert not (root / "other-key").exists() + assert _read_manifest(root)["entries"] == {} + + async def test_an_answer_under_another_key_does_not_overwrite_that_key( + self, root: Path + ) -> None: + """The aliased answer must not reach the real key's file. + + Both keys are requested here, so nothing is prunable and the write + itself is what is under test: unchecked, the object served under the + alias is written to the *other* key's path, clobbering the content that + key's own lookup resolved — and the run still reports ok. + """ + aliased = "aliased\n" + + class AliasingStore: + """Answers one key honestly and the other under that same key.""" + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + if key == "other-key": + return { + "key": "other-key", + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + return { + "key": "other-key", + "version": 2, + "content": aliased, + "contentHash": _hash(aliased), + } + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + existing = _place_managed(root, "other-key", SKILL_BODY) + skills_module._set_store(AliasingStore()) + + # The alias is resolved last, so an unchecked write lands on top. + report = await write_skills(["other-key", "requested-key"], root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert _actions_by_key(report)["requested-key"].action == "error" + assert _actions_by_key(report)["other-key"].action == "skipped_current" + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: _place_managed(root, "a", SKILL_BODY) before = _read_manifest(root) From 1cf523566fdfd0a6fc70ecc9eac5220ac52e2a45 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:46:56 -0400 Subject: [PATCH 21/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20name=20the=20payload=20a=20transfer=20completed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The protocol reader took payloads[0]'s intentCode and applied it to the skill object set, which is what the delivery protocol requires — one payload per credential, read the first intent, tolerate the rest — but it left the assumption behind that rule undocumented and unguarded. If the one-payload guarantee ever widens, an xfer-full for another payload would start an empty pending set and the next payload-transferred would publish it: every skill reported revoked, and with pruning on, a customer's files deleted. The first payload is still the payload that is read. What is new is that the reader now knows which payload skills actually arrive on — learnt from the intent's id, or from the (p::) selector, since no object or transfer event carries a payload id of its own — and declines to apply a transfer of any other, holding last known good, warning once, and counting it in diagnostics.payloads_ignored. An intent describing more than one payload warns once on its own, because that is the one case the comparison cannot catch: another payload's transfer arriving before any skill has been seen has nothing to be compared against. Behaviour under one-payload delivery is unchanged, and a full transfer of the skill payload still empties it — every skill deleted is a real state the guard must not mask. Co-Authored-By: Claude Opus 5 --- packages/client/agents.md | 13 + .../src/launchdarkly_ai_server/skills_fdv2.py | 132 +++++++++- packages/client/tests/test_skills_fdv2.py | 226 ++++++++++++++++++ 3 files changed, 370 insertions(+), 1 deletion(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 80ec0538..e73c96d4 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -256,6 +256,19 @@ described, and would briefly empty the store — which, with pruning on, is the between a reconcile and deleting a customer's skill files. An interrupted transfer therefore leaves last known good intact, and listeners fire once per commit. +**The first payload intent is read, and is assumed to be the skill payload.** Delivery +provides one payload per credential and the protocol requires a client to ignore all but the +first payload intent, so `payloads[0]` is both what arrives and what the protocol says to +read. The cost of that assumption is that an `xfer-full` for somebody *else's* payload would +start an empty pending set, and the next `payload-transferred` would publish it — every skill +reported revoked, and with pruning on, a customer's files deleted. `_ProtocolReader` +therefore learns which payload skills arrive on, from the intent's `id` or from the +`(p::)` selector, and declines to apply a transfer of any other: once at +WARNING, counted in `diagnostics.payloads_ignored`, holding last known good. A transfer that +names no payload is applied, since one-payload delivery is the common case. The residual is +the first transfer of a connection — before a skill has arrived there is nothing to compare +against — which is what the separate WARNING on a multi-payload intent is for. + **A hashless object is held, not dropped.** Verification withholds it with `missing_content_hash`; the transport's job is to make that loud (an error per object, a summary per wholly-hashless payload, `diagnostics.hashless_objects`) rather than to work diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 80ef1224..f644a349 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -36,6 +36,12 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. +One assumption it *does* make, and states: **the payload intent it reads is the +payload skills arrive on.** Delivery sends one payload per credential and the +protocol tells a client to read only the first payload intent, so today those are +the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the +cost of conflating them is an emptied skill set. + The design rationale — why ``objectVersion`` is not ``version``, why changes commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. @@ -93,6 +99,16 @@ very thing verification exists to check. """ +_PAYLOAD_SELECTOR = re.compile(r"\(p:([^:()]+):\d+\)") +""" +The payload identity inside a transfer's selector, ``(p::)``. + +The selector is the only place a completed transfer names its own payload: +``put-object``, ``delete-object`` and ``payload-transferred`` carry no payload id +of their own. ``_ProtocolReader`` reads it as a fallback for an intent that named +no ``id``. +""" + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -169,6 +185,11 @@ class StoreDiagnostics: future kind. Skipping is the contract, not a failure.""" objects_revoked: int = 0 """``delete-object`` events applied to skills.""" + payloads_ignored: int = 0 + """ + Transfers not applied because they completed a payload other than the one + skills arrive on. Zero while delivery sends one payload per connection. + """ hashless_objects: int = 0 """ Skill objects whose envelope carried no ``contentHash``. @@ -255,6 +276,22 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: return raw +def _payload_id_of(intent: Any) -> str | None: + """The payload id one payload intent names, when it names a usable one.""" + if not isinstance(intent, dict): + return None + value = intent.get("id") + return value if isinstance(value, str) and value else None + + +def _payload_id_from_selector(state: Any) -> str | None: + """The payload id inside a transfer's selector, when it carries one.""" + if not isinstance(state, str): + return None + match = _PAYLOAD_SELECTOR.search(state) + return match.group(1) if match else None + + def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with @@ -394,6 +431,18 @@ class _ProtocolReader: version is the unit of consistency: applying half of one would publish a state the server never described, and on a full transfer would briefly empty the store. Listeners therefore fire once per commit, not once per object. + + **The first payload intent is read, and is assumed to be the skill payload.** + Delivery provides one payload per credential and the protocol requires a + client to ignore all but the first payload intent, so ``payloads[0]`` is both + what arrives and what the protocol says to read. If that ever widens, an + ``xfer-full`` for somebody else's payload would empty the skill set and the + next ``payload-transferred`` would publish it empty — with pruning on, the + difference between a reconcile and deleting a customer's files. This layer + therefore learns which payload skills arrive on and declines to apply a + transfer of any other, once at WARNING and counted. The residual is the first + transfer of a connection: before a skill has arrived there is nothing to + compare a payload against. """ def __init__(self, committed: _SkillObjectSet) -> None: @@ -406,6 +455,14 @@ def __init__(self, committed: _SkillObjectSet) -> None: # recreated store reports again and two stores never quieten each other. # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() + # The payload the current intent describes, and the payload skills have + # actually arrived on. One payload per connection makes these the same + # payload; the class docstring says why they are kept apart regardless. + self._intent_payload_id: str | None = None + self._skill_payload_id: str | None = None + self._skills_in_payload = 0 + self._warned_multiple_payloads = False + self._warned_foreign_payload = False # -- events ------------------------------------------------------------ @@ -434,10 +491,15 @@ def _server_intent(self, data: Any) -> _TransferOutcome: return _TransferOutcome( disconnect="server-intent carried no payload description" ) + if len(payloads) > 1: + self._warn_multiple_payloads(payloads) + # The first payload only, as the protocol requires. first = payloads[0] intent = first.get("intentCode") if isinstance(first, dict) else None self._intent = intent + self._intent_payload_id = _payload_id_of(first) self._changes = [] + self._skills_in_payload = 0 if intent == _INTENT_TRANSFER_FULL: # Built alongside the live set rather than in place, so an # interrupted transfer leaves last known good intact. @@ -479,6 +541,7 @@ def _put_object(self, data: Any) -> _TransferOutcome: target.put(raw) self._changes.append(raw) self.diagnostics.skill_objects_received += 1 + self._skills_in_payload += 1 if not isinstance(raw.get("contentHash"), str): self.diagnostics.hashless_objects += 1 self._warn_hashless(raw) @@ -493,6 +556,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: return _TransferOutcome() target.delete(tombstone) self.diagnostics.objects_revoked += 1 + # A revocation identifies the payload as ours just as a put does. + self._skills_in_payload += 1 # A tombstone carries identity and no content, so a listener that reads # content must check for ``content`` rather than assume it. self._changes.append( @@ -503,11 +568,23 @@ def _delete_object(self, data: Any) -> _TransferOutcome: def _payload_transferred(self, data: Any) -> _TransferOutcome: state = data.get("state") if isinstance(data, dict) else None version = data.get("version") if isinstance(data, dict) else None - if self._pending is not None: + payload_id = self._intent_payload_id or _payload_id_from_selector(state) + if self._pending is not None and self._is_foreign_payload(payload_id): + self._warn_foreign_payload(payload_id) + self.diagnostics.payloads_ignored += 1 + self._changes = [] + elif self._pending is not None: self._committed.replace_with(self._pending) _warn_if_nothing_can_verify(self._committed) + if self._skills_in_payload and payload_id is not None: + # Learnt, not configured: nothing below the interface is told + # which payload is which, so the payload that carried a skill + # put or revocation is the payload skills arrive on. + self._skill_payload_id = payload_id self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 changes = self._changes self._changes = [] self.diagnostics.payloads_transferred += 1 @@ -526,6 +603,8 @@ def _abandon_in_flight(self) -> None: """Drops the in-flight payload and keeps what is committed.""" self._pending = None self._intent = None + self._intent_payload_id = None + self._skills_in_payload = 0 self._changes = [] def _error(self, data: Any) -> _TransferOutcome: @@ -546,8 +625,59 @@ def _goodbye(self, data: Any) -> _TransferOutcome: ) return _TransferOutcome(disconnect=f"server said goodbye: {reason}") + # -- payload identity ---------------------------------------------------- + + def _is_foreign_payload(self, payload_id: str | None) -> bool: + """ + Whether a transfer completes a payload other than the one skills arrive on. + + ``False`` unless both payloads are known, so one-payload delivery and the + first transfer of a connection behave exactly as they did before this + check existed. + """ + return ( + self._skill_payload_id is not None + and payload_id is not None + and payload_id != self._skill_payload_id + ) + # -- diagnostics --------------------------------------------------------- + def _warn_multiple_payloads(self, payloads: list[Any]) -> None: + """ + One WARNING per reader for an intent describing more than one payload. + + Not an error: reading only the first is what the protocol asks for. But it + means the first payload is no longer *guaranteed* to be the skill payload, + and an intent for another payload arriving before any skill has been seen + is the one case ``_is_foreign_payload`` cannot catch. + """ + if self._warned_multiple_payloads: + return + self._warned_multiple_payloads = True + logger.warning( + "An FDv2 server-intent described %d payloads (%s). Only the first is " + "read, as the protocol requires, and it is taken to be the payload " + "skills arrive on. If skills stop resolving from this point, that is " + "the assumption that broke; contact LaunchDarkly support.", + len(payloads), + ", ".join(str(_payload_id_of(p)) for p in payloads), + ) + + def _warn_foreign_payload(self, payload_id: str | None) -> None: + """One WARNING per reader for a transfer this layer declined to apply.""" + if self._warned_foreign_payload: + return + self._warned_foreign_payload = True + logger.warning( + "An FDv2 transfer of payload %s was not applied to the skills held, " + "which arrive on payload %s. Applying it would have replaced them " + "with whatever that payload carried — nothing, in the case of a flag " + "payload. The skills held are unchanged.", + payload_id, + self._skill_payload_id, + ) + def _warn_hashless(self, raw: dict[str, Any]) -> None: """ One ERROR per ``(key, version)`` whose envelope had no ``contentHash``. diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index ff5720bc..87596609 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -482,6 +482,232 @@ def test_an_object_arriving_with_no_intent_is_treated_as_a_delta(self) -> None: assert len(held) == 1 +# --------------------------------------------------------------------------- +# Which payload a transfer completed +# --------------------------------------------------------------------------- + + +def _payload_warnings(caplog: Any, fragment: str) -> list[Any]: + return [ + r + for r in caplog.records + if r.levelname == "WARNING" and fragment in r.getMessage() + ] + + +def skill_payload( + *object_events: tuple[str, Any], + payload_id: str = "agent-skill", + code: str = "xfer-full", + state: str = "basis-1", +) -> list[dict[str, Any]]: + """One payload's events, with the payload it belongs to named explicitly.""" + return events( + ("server-intent", server_intent(code, payload_id)), + *object_events, + ("payload-transferred", transferred(state)), + ) + + +class TestPayloadIdentity: + """ + Which payload a transfer completed, and why this layer tracks it at all. + + Delivery provides one payload per credential and the protocol requires a + client to read only the first payload intent, so today the payload read is + the payload skills arrive on. These assert the behaviour that survives if + the first of those stops holding: another payload's ``xfer-full`` must not + publish an empty skill set, because with pruning on that deletes a + customer's materialized files. + """ + + def test_only_the_first_payload_intent_is_read(self) -> None: + """Reading only the first is what the protocol asks for, however many + arrive — the point of the rest of this class is to make that safe.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + { + "id": "agent-skill", + "target": 1, + "intentCode": "xfer-full", + }, + {"id": "env-flags", "target": 2, "intentCode": "none"}, + ] + }, + ), + ("put-object", put_skill()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 1 + + def test_more_than_one_payload_intent_warns_once(self, caplog: Any) -> None: + reader = _ProtocolReader(_SkillObjectSet()) + intent = { + "payloads": [ + {"id": "env-flags", "target": 1, "intentCode": "xfer-changes"}, + {"id": "agent-skill", "target": 2, "intentCode": "xfer-changes"}, + ] + } + with caplog.at_level("WARNING"): + reader.handle("server-intent", intent) + reader.handle("server-intent", intent) + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + def test_one_payload_intent_warns_about_nothing(self, caplog: Any) -> None: + with caplog.at_level("WARNING"): + drive( + _ProtocolReader(_SkillObjectSet()), + skill_payload(("put-object", put_skill())), + ) + assert _payload_warnings(caplog, "payload") == [] + + def test_another_payloads_full_transfer_does_not_empty_the_skills_held( + self, caplog: Any + ) -> None: + """ + The case this guard exists for. A flag payload's ``xfer-full`` starts an + empty pending set; applying it at ``payload-transferred`` would publish + every skill as revoked, which a reconcile with pruning on reads as + "delete these files". + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + with caplog.at_level("WARNING"): + outcomes = drive( + reader, + skill_payload( + ("put-object", put_flag()), payload_id="env-flags", state="basis-2" + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + assert len(_payload_warnings(caplog, "was not applied")) == 1 + # Nothing changed, so no listener is woken to reconcile against it. + assert outcomes[-1].changes == [] + + def test_a_declined_transfer_warns_once_however_often_it_repeats( + self, caplog: Any + ) -> None: + """A polling connection sees the other payload on every poll.""" + reader = _ProtocolReader(_SkillObjectSet()) + drive(reader, skill_payload(("put-object", put_skill()))) + foreign = skill_payload(("put-object", put_flag()), payload_id="env-flags") + with caplog.at_level("WARNING"): + drive(reader, foreign) + drive(reader, foreign) + assert len(_payload_warnings(caplog, "was not applied")) == 1 + assert reader.diagnostics.payloads_ignored == 2 + + def test_a_full_transfer_of_the_skill_payload_still_empties_it(self) -> None: + """Every skill deleted is a real state, and the guard must not mask it.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive(reader, skill_payload(state="basis-2")) + assert len(held) == 0 + assert reader.diagnostics.payloads_ignored == 0 + + def test_a_revocation_identifies_the_payload_as_the_skill_payload(self) -> None: + """A payload that only revokes is still a payload skills arrive on.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive( + reader, + skill_payload(("delete-object", delete_skill()), code="xfer-changes"), + ) + drive( + reader, skill_payload(("put-object", put_skill()), payload_id="env-flags") + ) + assert reader.diagnostics.payloads_ignored == 1 + + def test_the_payload_is_identified_from_the_selector_when_no_id_is_named( + self, + ) -> None: + """``payload-transferred``'s selector is the only other place a completed + transfer names its payload.""" + held = _SkillObjectSet() + reader = _ProtocolReader(held) + unnamed = {"payloads": [{"target": 1, "intentCode": "xfer-full"}]} + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_skill()), + ("payload-transferred", transferred("(p:agent-skill:53)")), + ), + ) + drive( + reader, + events( + ("server-intent", unnamed), + ("put-object", put_flag()), + ("payload-transferred", transferred("(p:env-flags:12)")), + ), + ) + assert held.get("pdf-extraction", None) is not None + assert reader.diagnostics.payloads_ignored == 1 + + def test_an_unidentifiable_payload_is_applied_rather_than_withheld(self) -> None: + """ + A transfer naming no payload at all is the store's own, since delivery + sends it one payload. Withholding it would break the common case to + defend against a hypothetical one. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + drive(reader, skill_payload(("put-object", put_skill()))) + drive( + reader, + events( + ("server-intent", {"payloads": [{"intentCode": "xfer-full"}]}), + ("put-object", put_skill(object_version=4)), + ("payload-transferred", {"version": 44}), + ), + ) + assert held.get("pdf-extraction", None)["version"] == 4 + assert reader.diagnostics.payloads_ignored == 0 + + def test_the_first_transfer_of_a_connection_is_the_residual( + self, caplog: Any + ) -> None: + """ + Before a skill has arrived there is nothing to compare a payload + against, so another payload's ``xfer-full`` arriving first cannot be + told apart. The multiple-payload WARNING is the only signal there is, + which is why it exists. + """ + held = _SkillObjectSet() + reader = _ProtocolReader(held) + with caplog.at_level("WARNING"): + drive( + reader, + events( + ( + "server-intent", + { + "payloads": [ + {"id": "env-flags", "intentCode": "xfer-full"}, + {"id": "agent-skill", "intentCode": "xfer-full"}, + ] + }, + ), + ("put-object", put_flag()), + ("payload-transferred", transferred()), + ), + ) + assert len(held) == 0 + assert len(_payload_warnings(caplog, "described 2 payloads")) == 1 + + # --------------------------------------------------------------------------- # Interface parity with InMemorySkillStore # --------------------------------------------------------------------------- From efc4ca77daf50a878b5988a591613cd96b778fcb Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 15:24:45 -0400 Subject: [PATCH 22/43] =?UTF-8?q?fix(client):=20Agent=20Skills=20=E2=80=94?= =?UTF-8?q?=20read=20the=20skill's=20version=20off=20the=20wire=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK-facing FDv2 channel now delivers skills the way streamer #4681 and gonfalon #70638 spell them: object kinds are open strings, the agent-skill payload is classified `generic`, and every generic object carries only `key`, `kind`, `version` and `object`, exactly like a flag. A skill arrives under kind `skill` with its own version folded into the key as `:`. There is no `category` field and no `objectVersion` field; both came from an earlier streamer draft that never shipped. Identification is now the kind alone. The wire key is split in one place, `_split_wire_key`, and both the put and the delete translation go through it. A key that will not split cleanly is held rather than dropped — version-less, or with the offending text as its version — so verification withholds it with `invalid_version` under a key the caller recognises; only a key with nothing before the delimiter is dropped, since there is no identity to hold it under. `SDK_DATA_MODEL_VERSION` goes with it: the connection's `mv` parameter only accepts flag model versions, and generic payloads ignore it. The transport stops sending it in the following change. Co-Authored-By: Claude Fable 5.1 --- packages/client/agents.md | 46 ++--- .../src/launchdarkly_ai_server/skills_core.py | 6 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 159 ++++++++++++------ packages/client/tests/test_skills_fdv2.py | 130 ++++++++++---- 4 files changed, 233 insertions(+), 108 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index e73c96d4..978dc8ad 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the `objectVersion`/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -226,29 +226,37 @@ shape `skills_core.SkillStore` documents. It lives below the store interface; ** that interface knows it exists**. If a transport change ever seems to require editing an accessor, verification, or `write_skills`, the adapter boundary is wrong. -**`objectVersion` is the skill's version. `version` is the payload's.** On the wire a skill -`put-object` carries both, and they are not interchangeable: +**The skill's version is in the object's `key`. `version` is the payload's.** Each version +of a skill is its own object on the wire, identified as `:`: ```json -{"key":"pdf-extraction","kind":"inline-resource","category":"skill", - "objectVersion":3,"version":42, +{"key":"pdf-extraction:3","kind":"skill","version":42, "object":{"contentType":"text/markdown","content":"…","contentHash":"…","name":"…"}} ``` -`objectVersion` (3) is what a `{key, version}` reference pins and what becomes the stored -`version`. `version` (42) is the version of the *payload* the object arrived in — it moves -when anything in the environment moves, including a flag with nothing to do with skills. -Reading it as the skill's version fails **silently**: the object verifies, the hash matches, -and the caller gets content under a version number that means nothing. Flags and segments -carry only `version` and omit both `category` and `objectVersion`, which is exactly why the -two fields look interchangeable. `_store_object_from_put` is the only place the translation -happens, and `TestVersionTranslation` asserts it in both directions. - -**Skills are identified by `kind == "inline-resource" && category == "skill"`; everything else -is ignored, not rejected.** An environment's payload assignment carries its flag payload -alongside its agent-skill payload, so flag and segment objects arrive as a matter of course. -Erroring on an unrecognised kind would turn a normal payload into a permanent reconnect -loop — a flag-delivery outage caused by a skills rollout. +The `3` after the delimiter is what a `{key, version}` reference pins and what becomes the +stored `version`, under the stored key `pdf-extraction`. `version` (42) is the version of the +*payload* the object arrived in — it moves when anything in the environment moves, +including a flag with nothing to do with skills. Reading it as the skill's version fails +**silently**: the object verifies, the hash matches, and the caller gets content under a +version number that means nothing. There is no separate field for the skill's version: the +agent-skill payload is a *generic* payload, and generic objects carry only `key`, `kind`, +`version` and `object`, exactly like a flag. `_split_wire_key` is the only place the wire key +is read, `_store_object_from_put` and `_tombstone_from_delete` both go through it, and +`TestVersionTranslation` asserts the translation in both directions. A wire key that will +not split cleanly is *held*, not dropped — version-less, or with the offending text as its +version — so verification withholds it with `invalid_version` under a key the caller +recognises; only a key with nothing before the delimiter is dropped, since there is no +identity to hold it under. + +**Skills are identified by `kind == "skill"`; everything else is ignored, not rejected.** +Object kinds on the SDK-facing channel are open strings, and the agent-skill payload is +classified `generic`, so a skill arrives under the kind its producer registered — the bare +category name — not under a broader wrapper kind with a narrowing field. An environment's +payload assignment carries its flag payload alongside its agent-skill payload, so flag and +segment objects arrive as a matter of course. Erroring on an unrecognised kind would turn a +normal payload into a permanent reconnect loop — a flag-delivery outage caused by a skills +rollout. **Changes commit at `payload-transferred`, not as objects arrive.** A payload version is the unit of consistency: a half-applied full transfer would publish a state the server never diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 22166799..d2d870c4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -56,9 +56,9 @@ An **internal seam value**, deliberately not exported from the package root. It is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and a store adapter is free to map it onto -whatever the transport underneath actually uses — a delivery payload may well -carry skills under a broader kind with a narrower category, in which case -translating that pair to this one value is the adapter's job. +whatever the transport underneath actually uses — the value happens to match +the kind LaunchDarkly's delivery channel uses today, but a transport that spelt +it differently would translate, and that translation is the adapter's job. Exporting it would publish an SDK-side seam string as though it were the wire contract, which is a claim this side cannot make and would be hard to walk back diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index f644a349..d20a0e2f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -3,7 +3,7 @@ The half of the delivery transport that has no I/O: identifying skill objects on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, objectVersion)``, and applying a +interface defines, holding them by ``(key, version)``, and applying a payload's events as one consistent commit. ``FDv2SkillStore``, the store that puts a network connection underneath this, follows in a separate change. @@ -42,9 +42,9 @@ the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the cost of conflating them is an emptied skill set. -The design rationale — why ``objectVersion`` is not ``version``, why changes -commit at ``payload-transferred`` — is in ``agents.md`` under *The delivery -transport*. +The design rationale — why the skill's version is read from the object's +``key`` and never from ``version``, why changes commit at +``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. """ from __future__ import annotations @@ -63,21 +63,27 @@ # The wire contract # --------------------------------------------------------------------------- -FDV2_OBJECT_KIND = "inline-resource" +FDV2_OBJECT_KIND = "skill" """ -The FDv2 ``kind`` skills are delivered under. Together with -``FDV2_OBJECT_CATEGORY`` it maps onto the single interface value -``skills_core.SKILL_OBJECT_KIND``; that translation is this adapter's job. +The FDv2 ``kind`` skills are delivered under. + +Object kinds on the SDK-facing channel are open strings: the agent-skill payload +is classified ``generic`` and every object in it carries the kind its producer +registered, which for skills is the bare category name. Delivery lower-cases the +kind, so an exact comparison is the whole test. The kind happens to equal +``skills_core.SKILL_OBJECT_KIND`` today; they are still separate constants, +because one is a wire value LaunchDarkly owns and the other is an SDK seam. """ -FDV2_OBJECT_CATEGORY = "skill" -"""The ``category`` that narrows ``inline-resource`` to an agent skill.""" - -SDK_DATA_MODEL_VERSION = 1 +FDV2_KEY_DELIMITER = ":" """ -The ``mv`` request parameter. The one request parameter whose value could not be -confirmed against a live server, so treat the default as provisional and -override it through the store's ``data_model_version`` if needed. +What separates a skill's key from its version inside the object's wire ``key``. + +A generic object is identified on the wire as ``:`` — the skill's +own key, one delimiter, the skill's own version — because each version of a +skill is a distinct object in the payload. Delivery forbids the delimiter inside +a registered category and skill keys cannot contain it, so a well-formed wire key +has exactly one. """ _EVENT_SERVER_INTENT = "server-intent" @@ -204,7 +210,7 @@ class StoreDiagnostics: # --------------------------------------------------------------------------- -# Deserialisation — where objectVersion is not version +# Deserialisation — where the skill's version lives in the key, not in version # --------------------------------------------------------------------------- @@ -220,17 +226,58 @@ def _is_skill_event(data: Any) -> bool: """ Whether one ``put-object`` / ``delete-object`` payload is a skill. - Both halves are required: ``inline-resource`` may carry other categories, - and flags and segments omit ``category`` entirely. Every other kind is - ignored, not rejected, because flag and segment objects share the connection - and erroring on them would turn a normal payload into a reconnect loop. + The kind alone decides it. Every other kind is ignored, not rejected, + because flag and segment objects share the connection and erroring on them + would turn a normal payload into a reconnect loop. """ if not isinstance(data, dict): return False - return ( - data.get("kind") == FDV2_OBJECT_KIND - and data.get("category") == FDV2_OBJECT_CATEGORY - ) + return data.get("kind") == FDV2_OBJECT_KIND + + +@dataclass(frozen=True) +class _WireIdentity: + """A skill object's wire ``key``, split into the skill's key and version.""" + + key: str + version: Any + """``int`` when the wire carried one; the offending text when it did not; + absent (``_NO_VERSION``) when the wire key had no delimiter at all.""" + + +_NO_VERSION = object() + + +def _split_wire_key(wire_key: Any) -> _WireIdentity | None: + """ + Reads ``:`` off one object's wire ``key``. + + Lenient where leniency keeps the object diagnosable and strict only where + there is nothing to diagnose: + + - No delimiter: the whole wire key is the skill key and there is no version, + so the object is held version-less and verification reports + ``invalid_version`` under a key the caller can recognise. + - A version that is not a run of digits (``"pdf:latest"``, ``"pdf:"``, + ``"a:1:2"``): the text is carried through *as the version*, for the same + reason — the caller learns that ``pdf`` arrived broken, not that it is + absent. + - An empty key before the delimiter (``":3"``): there is no identity to hold + it under, so ``None``, and the caller drops it. + + Leading zeros are accepted (``"pdf:03"`` is version 3) since ``int`` is the + identity a reference pins, not the spelling. + """ + if not isinstance(wire_key, str) or not wire_key: + return None + key, delimiter, version_text = wire_key.partition(FDV2_KEY_DELIMITER) + if not key: + return None + if not delimiter: + return _WireIdentity(key=key, version=_NO_VERSION) + if version_text.isascii() and version_text.isdigit(): + return _WireIdentity(key=key, version=int(version_text)) + return _WireIdentity(key=key, version=version_text) def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: @@ -240,33 +287,36 @@ def _store_object_from_put(data: dict[str, Any]) -> dict[str, Any] | None: **The one translation this adapter must get right:** - wire ``objectVersion`` → stored ``version`` (the skill's own version) - wire ``version`` → dropped (the *payload* version) + wire ``key`` → stored ``key`` and ``version`` (split on ``:``) + wire ``version`` → dropped (the *payload* version) - ``objectVersion`` is what a ``{key, version}`` reference pins; ``version`` - moves whenever anything in the environment moves. Confusing them fails - silently: the object verifies and the caller gets content under a version - number that means nothing. + Each version of a skill is its own object on the wire, identified as + ``:``; that version is what a ``{key, version}`` reference + pins. The event's ``version`` field is the version of the payload the object + arrived in and moves whenever anything in the environment moves. Confusing + them fails silently: the object verifies and the caller gets content under a + version number that means nothing. - Returns ``None`` only when ``key`` is not a string, since a keyless object - has no identity to store it under. Every other defect is carried through - verbatim so that verification withholds it with a reason code rather than - the transport dropping it into indistinguishable absence. + Returns ``None`` only when the wire ``key`` carries no skill key at all, + since such an object has no identity to store it under. Every other defect + is carried through so that verification withholds it with a reason code + rather than the transport dropping it into indistinguishable absence. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill put-object carried no string 'key' and could not be " - "stored under any identity; it was dropped." + "An FDv2 skill put-object carried no usable 'key' (%r) and could not " + "be stored under any identity; it was dropped.", + data.get("key"), ) return None - raw: dict[str, Any] = {"key": key} + raw: dict[str, Any] = {"key": identity.key} - # A membership test rather than a `.get` default, so an explicitly-null - # objectVersion stays null and reaches verification as `invalid_version`. - if "objectVersion" in data: - raw["version"] = data["objectVersion"] + # Absent stays absent and malformed stays malformed, so verification sees + # what arrived (as `invalid_version`) rather than something invented here. + if identity.version is not _NO_VERSION: + raw["version"] = identity.version envelope = data.get("object") if isinstance(envelope, dict): @@ -294,25 +344,26 @@ def _payload_id_from_selector(state: Any) -> str | None: def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: """ - Narrows one FDv2 skill ``delete-object`` to the identity it revokes, with - the same ``objectVersion`` translation as a put. + Narrows one FDv2 skill ``delete-object`` to the identity it revokes, reading + the wire ``key`` the same way a put does. An ``object_version`` of ``None`` means the delete named no usable version and is read as "revoke every version of this key". That is the safe direction: the alternative is continuing to serve content LaunchDarkly has - withdrawn. + withdrawn. It also removes whatever a malformed put of the same wire key + left held, since that was stored version-less under the same skill key. """ - key = data.get("key") - if not isinstance(key, str) or not key: + identity = _split_wire_key(data.get("key")) + if identity is None: logger.warning( - "An FDv2 skill delete-object carried no string 'key'; it was ignored." + "An FDv2 skill delete-object carried no usable 'key' (%r); it was ignored.", + data.get("key"), ) return None - object_version = data.get("objectVersion") return _Tombstone( - key=key, - object_version=object_version - if is_valid_skill_version(object_version) + key=identity.key, + object_version=identity.version + if is_valid_skill_version(identity.version) else None, ) @@ -324,7 +375,7 @@ def _tombstone_from_delete(data: dict[str, Any]) -> _Tombstone | None: class _SkillObjectSet: """ - Raw skill objects held in memory, keyed by ``(key, objectVersion)``. + Raw skill objects held in memory, keyed by ``(key, version)``. Lookup semantics are identical to ``InMemorySkillStore``'s, down to the fall-through to a version-less entry, so that the store a caller configures diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 87596609..b88d14d9 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,8 +1,8 @@ """ Tests for the FDv2 skill delivery protocol. -Wire semantics — which objects are skills, ``objectVersion`` versus ``version``, -revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted +Wire semantics — which objects are skills, the skill's version in the wire ``key`` +versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted against ``_ProtocolReader``, which has no I/O, so each case reads as the contract it is rather than as a server script. """ @@ -17,7 +17,7 @@ from launchdarkly_ai_server import InMemorySkillStore from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( - FDV2_OBJECT_CATEGORY, + FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, _is_skill_event, _ProtocolReader, @@ -40,6 +40,18 @@ def _hash(content: str) -> str: # --------------------------------------------------------------------------- +def wire_key(key: str, object_version: Any) -> str: + """ + The wire ``key`` of one skill object: ``:``. + + ``None`` builds a key with no version at all, which is how the tests spell a + malformed object; anything else is spelled after the delimiter verbatim. + """ + if object_version is None: + return key + return f"{key}{FDV2_KEY_DELIMITER}{object_version}" + + def put_skill( key: str = "pdf-extraction", *, @@ -62,10 +74,8 @@ def put_skill( content_hash if content_hash is not None else _hash(content) ) return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, "object": envelope, } @@ -75,16 +85,14 @@ def delete_skill( key: str = "pdf-extraction", *, object_version: Any = 3, payload_version: int = 43 ) -> dict[str, Any]: return { - "key": key, + "key": wire_key(key, object_version), "kind": FDV2_OBJECT_KIND, - "category": FDV2_OBJECT_CATEGORY, - "objectVersion": object_version, "version": payload_version, } def put_flag(key: str = "my-flag", version: int = 17) -> dict[str, Any]: - """A flag ``put-object``: no ``category``, no ``objectVersion``.""" + """A flag ``put-object``: the same envelope fields, a different ``kind``.""" return { "key": key, "kind": "flag", @@ -141,30 +149,37 @@ def full_payload( class TestObjectIdentification: - def test_kind_and_category_together_identify_a_skill(self) -> None: + def test_the_kind_alone_identifies_a_skill(self) -> None: assert _is_skill_event(put_skill()) is True + def test_the_kind_is_the_bare_category_name(self) -> None: + """ + Object kinds on the channel are open strings and the agent-skill payload + is ``generic``, so a skill arrives under the kind its producer + registered — ``skill`` — not under a broader wrapper kind. + """ + assert FDV2_OBJECT_KIND == "skill" + def test_a_flag_is_not_a_skill(self) -> None: assert _is_skill_event(put_flag()) is False def test_a_segment_is_not_a_skill(self) -> None: assert _is_skill_event(put_segment()) is False - def test_inline_resource_of_another_category_is_not_a_skill(self) -> None: - """``inline-resource`` is a broad kind, so the category is required too.""" + def test_another_generic_kind_is_not_a_skill(self) -> None: + """A generic payload may carry other registered kinds one day.""" other = put_skill() - other["category"] = "prompt-template" + other["kind"] = "prompt-template" assert _is_skill_event(other) is False - def test_skill_category_under_another_kind_is_not_a_skill(self) -> None: + def test_a_skill_shaped_envelope_under_another_kind_is_not_a_skill(self) -> None: other = put_skill() other["kind"] = "some-future-kind" assert _is_skill_event(other) is False - def test_a_flag_shaped_object_with_no_category_is_not_a_skill(self) -> None: - """Flags and segments omit ``category`` entirely — the documented shape.""" - assert "category" not in put_flag() - assert "objectVersion" not in put_flag() + def test_nothing_but_the_kind_is_consulted(self) -> None: + """No secondary field narrows the kind, and none may be required.""" + assert set(put_skill()) == {"key", "kind", "version", "object"} @pytest.mark.parametrize("value", [None, "skill", 3, [], ()]) def test_non_dict_events_are_not_skills(self, value: Any) -> None: @@ -172,15 +187,27 @@ def test_non_dict_events_are_not_skills(self, value: Any) -> None: # --------------------------------------------------------------------------- -# objectVersion is not version +# The skill's version is in the wire key; `version` is the payload's # --------------------------------------------------------------------------- class TestVersionTranslation: - def test_object_version_becomes_the_seam_version(self) -> None: + def test_the_wire_key_is_key_colon_version(self) -> None: + assert ( + put_skill("pdf-extraction", object_version=3)["key"] == "pdf-extraction:3" + ) + + def test_the_version_after_the_delimiter_becomes_the_seam_version(self) -> None: raw = _store_object_from_put(put_skill(object_version=3, payload_version=42)) assert raw is not None assert raw["version"] == 3 + assert isinstance(raw["version"], int) + + def test_the_key_before_the_delimiter_becomes_the_seam_key(self) -> None: + """A caller asks for ``pdf-extraction``, never for ``pdf-extraction:3``.""" + raw = _store_object_from_put(put_skill("pdf-extraction", object_version=3)) + assert raw is not None + assert raw["key"] == "pdf-extraction" def test_the_payload_version_never_reaches_the_seam(self) -> None: """ @@ -200,36 +227,75 @@ def test_the_two_are_distinguished_even_when_the_payload_version_is_lower( assert raw is not None assert raw["version"] == 99 - def test_a_missing_object_version_is_not_defaulted_from_the_payload(self) -> None: - wire = put_skill() - del wire["objectVersion"] - raw = _store_object_from_put(wire) + def test_a_key_with_no_delimiter_is_held_version_less(self) -> None: + """Not defaulted from the payload version, and not dropped: verification + reports ``invalid_version`` under a key the caller recognises.""" + raw = _store_object_from_put(put_skill(object_version=None)) assert raw is not None + assert raw["key"] == "pdf-extraction" assert "version" not in raw - def test_an_explicitly_null_object_version_is_carried_through_as_null(self) -> None: - """Carried, not invented: verification reports ``invalid_version``.""" - raw = _store_object_from_put(put_skill(object_version=None)) + @pytest.mark.parametrize("spelling", ["latest", "", "3.0", "-1", "1:2", "３"]) + def test_a_version_that_is_not_digits_is_carried_through_as_invalid( + self, spelling: str + ) -> None: + """Carried, not invented: verification reports ``invalid_version`` for + the object rather than the transport reporting it absent.""" + raw = _store_object_from_put(put_skill(object_version=spelling)) + assert raw is not None + assert raw["key"] == "pdf-extraction" + assert raw["version"] == spelling + + def test_leading_zeros_spell_the_same_version(self) -> None: + raw = _store_object_from_put(put_skill(object_version="03")) assert raw is not None - assert raw["version"] is None + assert raw["version"] == 3 - def test_a_delete_translates_object_version_too(self) -> None: + def test_a_delete_reads_the_wire_key_the_same_way(self) -> None: tombstone = _tombstone_from_delete( delete_skill(object_version=3, payload_version=43) ) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version == 3 - def test_a_delete_with_no_usable_object_version_revokes_every_version(self) -> None: - tombstone = _tombstone_from_delete(delete_skill(object_version=None)) + @pytest.mark.parametrize("spelling", [None, "latest", "0"]) + def test_a_delete_with_no_usable_version_revokes_every_version( + self, spelling: Any + ) -> None: + tombstone = _tombstone_from_delete(delete_skill(object_version=spelling)) assert tombstone is not None + assert tombstone.key == "pdf-extraction" assert tombstone.object_version is None + @pytest.mark.parametrize("bad_key", [":3", "", None, 3]) + def test_a_put_with_no_skill_key_is_dropped_because_it_has_no_identity( + self, bad_key: Any + ) -> None: + wire = put_skill() + wire["key"] = bad_key + assert _store_object_from_put(wire) is None + def test_a_keyless_put_is_dropped_because_it_has_no_identity(self) -> None: wire = put_skill() del wire["key"] assert _store_object_from_put(wire) is None + def test_a_delete_with_no_skill_key_is_ignored(self) -> None: + wire = delete_skill() + wire["key"] = ":3" + assert _tombstone_from_delete(wire) is None + + def test_the_stored_identity_round_trips_to_the_wire_key(self) -> None: + """``_SkillObjectSet.snapshot`` spells its opaque keys the way the wire + does, so a held object can be matched back to the event that carried it.""" + held = _SkillObjectSet() + wire = put_skill("pdf-extraction", object_version=3) + raw = _store_object_from_put(wire) + assert raw is not None + held.put(raw) + assert set(held.snapshot()) == {wire["key"]} + def test_the_envelope_is_copied_verbatim(self) -> None: raw = _store_object_from_put(put_skill()) assert raw is not None From b6a25f9f455258dabb8d13b13978123239327376 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Thu, 10 Sep 2026 16:05:04 -0400 Subject: [PATCH 23/43] =?UTF-8?q?feat(client):=20Agent=20Skills=20?= =?UTF-8?q?=E2=80=94=20the=20FDv2=20delivery=20transport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FDv2SkillStore puts LaunchDarkly's SDK-facing FDv2 channel underneath the protocol layer: GET /sdk/poll and GET /sdk/stream, authenticated with the environment's server-side SDK key, streaming by default. It carries basis across requests, sends If-None-Match and treats 304 as a current answer, retries with capped jittered backoff, honours Retry-After only up to max_backoff, gives up after a bounded run of consecutive failures where a committed payload resets the count, and keeps serving last known good through every failure. A mobile key or client-side environment ID is refused in the constructor. Standard library only. close interrupts the socket rather than only setting a flag, because the delivery thread lives in a read no flag can reach; without that every shutdown of a healthy stream waited out the full join timeout. The no-store message now names FDv2SkillStore first, and watch_skills points at it as the store with a delivery transport. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 62 + packages/client/agents.md | 25 +- .../src/launchdarkly_ai_server/__init__.py | 5 +- .../src/launchdarkly_ai_server/skills_core.py | 13 +- .../src/launchdarkly_ai_server/skills_fdv2.py | 748 ++++++++- .../launchdarkly_ai_server/skills_watch.py | 2 +- packages/client/tests/test_skills.py | 12 + packages/client/tests/test_skills_fdv2.py | 1332 ++++++++++++++++- packages/client/tests/test_skills_watch.py | 5 +- 9 files changed, 2162 insertions(+), 42 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 41d5c6b3..42b1f40d 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -445,6 +445,66 @@ which OS ran the write. The keys stay valid everywhere else: an AI Config refere named `aux` parses, and its other fields are unaffected. If you have a skill named for a device, rename it. +#### Receiving skills from LaunchDarkly + +`InMemorySkillStore` is for tests and bring-your-own-content. In production, skill content +arrives through `FDv2SkillStore`, which speaks LaunchDarkly's SDK-facing FDv2 delivery +channel — the same `GET /sdk/poll` and `GET /sdk/stream` endpoints the base SDK's FDv2 data +source uses, authenticated with the environment's server-side SDK key. + +```python +import os + +from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +store.wait_for_skills(timeout=10) +await init_client(options={"skillStore": store}) + +# Materialize now, and re-materialize whenever delivery changes. +report, watcher = await watch_skills("*", ".claude/skills") +try: + ... +finally: + watcher.close() + store.close() +``` + +**Nothing above the store changes.** The accessors, verification, and `write_skills` see raw +objects through the `SkillStore` interface and cannot tell which store produced them. + +**Server-side only.** Skills are for server-side agent runtimes and skill content is +customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the +constructor. + +**Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches +a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired +with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an +outage the store keeps serving the last content it received and `write_skills`' default +`on_unavailable="keep"` leaves managed files alone — an outage must not read as "everything +was revoked". + +**One network timeout, and its default depends on the mode.** `read_timeout` bounds every +socket operation of a request, connecting included. In `mode="poll"` it bounds the whole +request and defaults to 10 seconds; in `mode="stream"` it bounds each wait for the next bytes +and defaults to 300 seconds, well beyond LaunchDarkly's heartbeat interval. + +**The connection also carries your flags.** A client cannot request only the skill payload, +so a skills-enabled environment delivers flag and segment objects on the same connection. +They are skipped, not evaluated — this store does no evaluation of any kind — and +`diagnostics.objects_ignored` counts them. + +> **Beta caveats, worth knowing before you deploy.** Payload signing does not exist on this +> channel yet, so delivery is TLS-only and the content hash establishes self-consistency, not +> origin authenticity. The FDv2 protocol is opt-in per account: without it the endpoints +> return HTTP 403, which the store reports as a fatal error explaining what to do. `ld-relay` +> does not speak the FDv2 endpoints, so relay-only deployments cannot receive skills. + +**If every skill comes back empty, check `diagnostics.hashless_objects`.** Verification +withholds any delivered object without a `contentHash`, so a nonzero count means skills are +being withheld rather than that the environment has none. The store also logs an error per +hashless object naming the reason. There is deliberately no fallback that skips verification. + **Total path length is yours to bound, not the SDK's.** The 255-byte bound above is per *component*; the root is your path, so `` + `` + `/SKILL.md` can still exceed Windows' 260-character `MAX_PATH` with a perfectly legal key. Choose a short managed root on @@ -460,7 +520,9 @@ Windows. | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | | `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | | `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | +| `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | Configure the store with `init_client(options={"skillStore": store})`. With none configured, the accessors raise `RuntimeError` explaining what to do and `write_skills` reports the diff --git a/packages/client/agents.md b/packages/client/agents.md index 978dc8ad..2a19c273 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -30,7 +30,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/types_validation.py` | `parse_ai_config` — validates flag variation shape; `is_valid_skill_key` / `is_valid_skill_version` / `skill_key_rejection_reason` (the canonical key-grammar explanation every layer quotes) | | `src/launchdarkly_ai_server/skills.py` | Agent Skills, retrieval half — `skill_refs`, `get_skill`/`get_skills`/`all_skills`, `InMemorySkillStore`, and the store/telemetry injection points `_set_store` / `_set_emitter_for_testing` | | `src/launchdarkly_ai_server/skills_core.py` | Shared skills internals — the `SkillStore` seam, module state, the telemetry seam and its three recorders, integrity verification, and store resolution. Imported by both `skills.py` and the materialization layer; imports neither | -| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery protocol — the wire-key/`version` translation, the held object set, and the pure `_ProtocolReader` that commits a payload's events at `payload-transferred`. Sits **below** the store interface; nothing in the feature imports it | +| `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | | `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | @@ -221,10 +221,12 @@ of skills, and the `"*"` reconcile, since `//SKILL.md` is a single pa ### The delivery transport, and the one field that will bite you -`skills_fdv2.py` translates LaunchDarkly's FDv2 delivery protocol into raw objects in the -shape `skills_core.SkillStore` documents. It lives below the store interface; **nothing above -that interface knows it exists**. If a transport change ever seems to require editing an -accessor, verification, or `write_skills`, the adapter boundary is wrong. +`FDv2SkillStore` speaks LaunchDarkly's SDK-facing FDv2 channel (`GET /sdk/poll`, +`GET /sdk/stream`, server-side SDK key in `Authorization`, `basis` + `mv` params, +`If-None-Match`/304). It lives below the store interface and produces raw objects in the +shape `skills_core.SkillStore` documents; **nothing above that interface knows it exists**. If a transport +change ever seems to require editing an accessor, verification, or `write_skills`, the adapter +boundary is wrong. **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: @@ -285,6 +287,19 @@ such skill" — and would let a prune delete the last known-good copy on disk. N a hash from the delivered content: that certifies the content against itself and verifies nothing. +**There is one network timeout, not two.** `urllib`'s `timeout` is the socket timeout for the +whole operation, so connect, headers and each read share it, and the module cannot bound the +connect separately without a custom connection class it should not carry. `read_timeout` is +therefore the only knob, and its default is per mode (`DEFAULT_POLL_TIMEOUT` for a whole poll +request, `DEFAULT_STREAM_READ_TIMEOUT` for the gap between reads on a stream). Do not add a +parameter that the standard library cannot honour; `TestTimeouts` measures the bound against a +socket that accepts and never answers. + +**`close` interrupts the socket, it does not just set a flag.** The delivery thread spends its +life blocked in a read that no flag can reach, and closing a response from another thread does +not unblock CPython's buffered reader. `_interrupt_read` shuts the socket down underneath it. +Without that, every shutdown of a *healthy* stream blocks for the full join timeout. + ### The reported outcome vocabulary, and the `Resolution` mapping `get_skill` returns `Skill | None`; `get_skill_result` returns a frozen `SkillOutcome` diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index f863d847..0f987117 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -46,6 +46,7 @@ skill_refs, ) from .skills_core import SkillStore +from .skills_fdv2 import FDv2SkillStore, StoreDiagnostics from .skills_fs import ( MANIFEST_FILENAME, MANIFEST_VERSION, @@ -238,7 +239,9 @@ "write_skills", "SkillStore", "InMemorySkillStore", - # skills — the eager re-reconcile + # skills — the FDv2 delivery transport, and the eager re-reconcile it enables + "FDv2SkillStore", + "StoreDiagnostics", "watch_skills", "SkillWatcher", # skills — the three closed-set unions a typed consumer needs to name diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index d2d870c4..b8043012 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -134,9 +134,18 @@ NO_STORE_MESSAGE = ( "No skill store is configured, so skill content cannot be retrieved. Configure " - 'one with init_client(options={"skillStore": store}) — InMemorySkillStore is ' - "available for local development and testing." + 'one with init_client(options={"skillStore": store}) — FDv2SkillStore receives ' + "content from LaunchDarkly, and InMemorySkillStore is available for local " + "development and testing." ) +""" +The first thing a user sees when no store is configured, so it names both stores. + +``FDv2SkillStore`` comes first because it is the answer in production, and a +message that offered only ``InMemorySkillStore`` would point a deployment at the +development store. Callers match on "skill store"; keep that phrase if the +wording changes. +""" # --------------------------------------------------------------------------- diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index d20a0e2f..da84a3b4 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1,15 +1,10 @@ """ -Agent Skills — the FDv2 delivery protocol. +Agent Skills — the FDv2 delivery transport. -The half of the delivery transport that has no I/O: identifying skill objects -on the wire, translating them into the raw object shape the ``SkillStore`` -interface defines, holding them by ``(key, version)``, and applying a -payload's events as one consistent commit. ``FDv2SkillStore``, the store that -puts a network connection underneath this, follows in a separate change. - -It sits *below* the ``SkillStore`` interface, and everything above — the -accessors, integrity verification, the ``Skill`` dataclass, materialization — -is unaware of it. +The store implementation that talks to LaunchDarkly. It sits *below* the +``SkillStore`` interface: it produces raw wire objects in the shape +``skills_core`` documents, and everything above — the accessors, integrity +verification, the ``Skill`` dataclass, materialization — is unaware of it. Layering:: @@ -20,12 +15,12 @@ GET /sdk/poll, GET /sdk/stream, authenticated with the environment's server-side SDK key -Dependencies run one way: this module imports nothing from the feature beyond -the version validator in ``types_validation``, and nothing in the feature -imports it. It uses only the standard library, so it adds no dependency +Dependencies run one way: this module imports ``skills_core`` for the +interface's kind constant and nothing else from the feature, and nothing in the +feature imports it. It uses only the standard library, so it adds no dependency to a package whose sole runtime dependency is ``opentelemetry-api``. -Three things this layer does *not* do, on purpose: +Three things this module does *not* do, on purpose: - **It does not verify content.** Verification lives at the accessor boundary in ``skills_core`` so that it applies to every store equally, including a @@ -36,24 +31,29 @@ - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. -One assumption it *does* make, and states: **the payload intent it reads is the -payload skills arrive on.** Delivery sends one payload per credential and the -protocol tells a client to read only the first payload intent, so today those are -the same payload. ``_ProtocolReader`` keeps the pair apart anyway, because the -cost of conflating them is an emptied skill set. - The design rationale — why the skill's version is read from the object's ``key`` and never from ``version``, why changes commit at -``payload-transferred`` — is in ``agents.md`` under *The delivery transport*. +``payload-transferred``, why there is one network timeout — is in +``agents.md`` under *The delivery transport*. """ from __future__ import annotations +import json import logging +import math +import random import re +import socket +import threading +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal +from .skills_core import SKILL_OBJECT_KIND from .types_validation import is_valid_skill_version logger = logging.getLogger(__name__) @@ -86,6 +86,20 @@ has exactly one. """ +DEFAULT_BASE_URI = "https://sdk.launchdarkly.com" +"""Where the SDK-facing FDv2 endpoints live. Overridable for Federal and private +instances.""" + +POLL_PATH = "/sdk/poll" +STREAM_PATH = "/sdk/stream" + +DEFAULT_POLL_TIMEOUT = 10.0 +"""Default ``read_timeout`` in ``"poll"`` mode: the bound on one whole request.""" + +DEFAULT_STREAM_READ_TIMEOUT = 300.0 +"""Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated +between two reads. LaunchDarkly's heartbeats arrive well inside this.""" + _EVENT_SERVER_INTENT = "server-intent" _EVENT_PUT_OBJECT = "put-object" _EVENT_DELETE_OBJECT = "delete-object" @@ -115,6 +129,8 @@ no ``id``. """ +Mode = Literal["stream", "poll"] + _MOBILE_KEY_PREFIX = "mob-" _SERVER_KEY_PREFIX = "sdk-" _CLIENT_SIDE_ID = re.compile(r"\A[0-9a-f]{20,}\Z") @@ -504,7 +520,6 @@ def __init__(self, committed: _SkillObjectSet) -> None: self.diagnostics = StoreDiagnostics() # Identities already reported by ``_warn_hashless``. Per reader, so a # recreated store reports again and two stores never quieten each other. - # No lock: ``handle`` runs only on its owner's single delivery thread. self._warned_hashless: set[tuple[str, Any]] = set() # The payload the current intent describes, and the payload skills have # actually arrived on. One payload per connection makes these the same @@ -609,8 +624,8 @@ def _delete_object(self, data: Any) -> _TransferOutcome: self.diagnostics.objects_revoked += 1 # A revocation identifies the payload as ours just as a put does. self._skills_in_payload += 1 - # A tombstone carries identity and no content, so a listener that reads - # content must check for ``content`` rather than assume it. + # A tombstone carries identity and no content; see + # ``FDv2SkillStore.add_listener`` for what listeners should expect. self._changes.append( {"key": tombstone.key, "version": tombstone.object_version} ) @@ -780,3 +795,686 @@ def _warn_if_nothing_can_verify(committed: _SkillObjectSet) -> None: len(held), _HASHLESS_ADVICE, ) + + +# --------------------------------------------------------------------------- +# HTTP +# --------------------------------------------------------------------------- + + +class _FatalTransportError(Exception): + """A failure retrying cannot fix: bad credential, forbidden, wrong URI.""" + + +class _RecoverableTransportError(Exception): + """A failure worth retrying. Carries a server-requested delay when given one.""" + + def __init__(self, message: str, retry_after: float | None = None) -> None: + super().__init__(message) + self.retry_after = retry_after + + +_FORBIDDEN_ADVICE = ( + "The FDv2 protocol is opt-in per LaunchDarkly account and is served as HTTP " + "403 while it is off. Skill delivery needs it enabled; contact LaunchDarkly " + "support to enable it for your account." +) + + +def _retry_after_seconds(headers: Any) -> float | None: + """ + ``Retry-After`` in seconds, when the server sent a usable one. + + The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that + ``float`` accepts, fall back to our own backoff: none of them is a delay, + and an infinite one would overflow the wait that honours it. + """ + if headers is None: + return None + try: + raw = headers.get("Retry-After") + except AttributeError: + return None + if raw is None: + return None + try: + seconds: float = float(str(raw).strip()) + except ValueError: + return None + if not math.isfinite(seconds): + return None + return max(0.0, seconds) + + +def _classify_status(status: int, headers: Any) -> Exception: + """Turns an HTTP error status into the right exception type.""" + if status == 401: + return _FatalTransportError( + "LaunchDarkly rejected the SDK key (HTTP 401). Skill delivery cannot " + "start. Check that the key is the environment's server-side SDK key." + ) + if status == 403: + return _FatalTransportError( + f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" + ) + if status in (400, 405, 406, 414, 501): + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " + "The request this adapter sent was not understood. It carries only " + "the SDK key and, after the first payload, a 'basis' selector, so " + "check the base URI and that the endpoint speaks FDv2." + ) + return _RecoverableTransportError( + f"LaunchDarkly returned HTTP {status}", _retry_after_seconds(headers) + ) + + +def _interrupt_read(response: Any) -> None: + """ + Best-effort interruption of a read blocked on *response*, from another thread. + + Closing the response is not enough: CPython's buffered reader stays parked in + ``readline`` until bytes arrive. Shutting the *socket* down underneath it + unblocks it immediately. Reaching the socket means walking urllib's private + attribute chain, so every step is guarded and failure is silent: the + delivery thread is a daemon and ``close``'s join timeout is the backstop. + """ + for path in (("fp", "raw", "_sock"), ("fp", "_sock"), ("_sock",)): + found: Any = response + for name in path: + found = getattr(found, name, None) + if found is None: + break + if found is not None and hasattr(found, "shutdown"): + try: + found.shutdown(socket.SHUT_RDWR) + except OSError: + pass + return + + +class _StreamConnection: + """ + One open streaming connection: an event iterator plus a way to interrupt it + from another thread, which is what ``FDv2SkillStore.close`` needs. + """ + + def __init__(self, response: Any) -> None: + self._response = response + self.events = _iter_sse(response) + + def close(self) -> None: + """Interrupts the read. Safe to call from any thread, and twice.""" + _interrupt_read(self._response) + try: + self._response.close() + except Exception: + pass + + +@dataclass(frozen=True) +class _PollResult: + not_modified: bool + events: list[tuple[str, Any]] + etag: str | None + + +class _Requester: + """ + The only place this module opens a socket. Standard library only, on purpose. + + *read_timeout* is applied to every socket operation of a request. ``urllib`` + has no separate connect timeout: its ``timeout`` becomes the socket timeout + for the whole operation, so connecting, waiting for headers and each body + read are all bounded by the same value. + """ + + def __init__( + self, + sdk_key: str, + base_uri: str, + *, + read_timeout: float, + opener: Any = None, + ) -> None: + self._sdk_key = sdk_key + self._base_uri = base_uri.rstrip("/") + self._read_timeout = read_timeout + # Injectable so tests can drive a fake endpoint without a socket. + self._opener = opener or urllib.request.build_opener() + + def _url(self, path: str, basis: str | None) -> str: + """ + The request URL: the path, plus ``basis`` once a payload has committed. + + Deliberately no ``mv`` (data model version). That parameter selects the + *flag* data model and the connection rejects any value but the flag + default; the agent-skill payload is generic, is served regardless of it, + and has no model version of its own to ask for. + """ + if not basis: + return f"{self._base_uri}{path}" + return f"{self._base_uri}{path}?{urllib.parse.urlencode({'basis': basis})}" + + def _request( + self, path: str, basis: str | None, headers: dict[str, str] + ) -> urllib.request.Request: + all_headers = {"Authorization": self._sdk_key, **headers} + return urllib.request.Request( + self._url(path, basis), headers=all_headers, method="GET" + ) + + def poll(self, basis: str | None, etag: str | None) -> _PollResult: + """One ``GET /sdk/poll``. A 304 is a first-class outcome, not an error.""" + headers = {"Accept": "application/json"} + if etag: + headers["If-None-Match"] = etag + request = self._request(POLL_PATH, basis, headers) + try: + with self._opener.open(request, timeout=self._read_timeout) as response: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + except urllib.error.HTTPError as exc: + if exc.code == 304: + # urllib raises on 304 when no redirect handler swallows it. + return _PollResult(not_modified=True, events=[], etag=etag) + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"polling request failed: {type(exc).__name__}: {exc}" + ) from exc + + return _PollResult( + not_modified=False, events=_decode_poll_body(body), etag=new_etag + ) + + def stream(self, basis: str | None) -> _StreamConnection: + """Opens ``GET /sdk/stream``.""" + request = self._request( + STREAM_PATH, + basis, + {"Accept": "text/event-stream", "Cache-Control": "no-cache"}, + ) + try: + response = self._opener.open(request, timeout=self._read_timeout) + except urllib.error.HTTPError as exc: + raise _classify_status(exc.code, exc.headers) from exc + except Exception as exc: + raise _RecoverableTransportError( + f"streaming request failed: {type(exc).__name__}: {exc}" + ) from exc + return _StreamConnection(response) + + +def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: + """ + Unwraps ``{"events": [...]}``. Polling and streaming carry identical event + objects, which is why the protocol reader is shared between the two modes. + """ + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _RecoverableTransportError( + f"polling response was not valid JSON: {exc}" + ) from exc + if not isinstance(parsed, dict) or not isinstance(parsed.get("events"), list): + raise _RecoverableTransportError("polling response had no 'events' array") + events: list[tuple[str, Any]] = [] + for entry in parsed["events"]: + if not isinstance(entry, dict): + continue + name = entry.get("event") + if isinstance(name, str): + events.append((name, entry.get("data"))) + return events + + +def _iter_sse(response: Any) -> Any: + """ + Decodes an SSE body into ``(event name, data)`` pairs. + + Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined + with newlines, a blank line dispatching, and ``:`` comments skipped. + """ + try: + name: str | None = None + data_lines: list[str] = [] + for raw_line in response: + line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") + if line == "": + if name is not None: + payload = "\n".join(data_lines) + try: + parsed = json.loads(payload) if payload else None + except json.JSONDecodeError: + logger.warning( + "Discarding FDv2 '%s' event whose data was not JSON", name + ) + parsed = None + else: + yield name, parsed + name = None + data_lines = [] + continue + if line.startswith(":"): + continue + field_name, _, value = line.partition(":") + value = value[1:] if value.startswith(" ") else value + if field_name == "event": + name = value + elif field_name == "data": + data_lines.append(value) + finally: + try: + response.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +def _backoff_delay( + attempt: int, *, base: float, maximum: float, jitter: float = 0.5 +) -> float: + """ + Exponential backoff with jitter, capped at *maximum*. + + Jitter is subtractive over the whole range rather than added on top, so the + cap is a real ceiling: a fleet restarted together must not reconnect in + lockstep, and must not exceed the interval the cap promises. + """ + # float(2 ** n): the integer power is untyped to mypy. + ceiling: float = min(maximum, base * float(2 ** max(0, attempt - 1))) + return ceiling * (1.0 - jitter * random.random()) + + +# --------------------------------------------------------------------------- +# The store +# --------------------------------------------------------------------------- + + +class FDv2SkillStore: + """ + A ``SkillStore`` fed by LaunchDarkly's SDK-facing FDv2 delivery channel. + + Constructed with the environment's server-side SDK key, started explicitly, + and passed to ``init_client``:: + + store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) + store.start() + store.wait_for_skills(timeout=10) + await init_client(options={"skillStore": store}) + + skill = await get_skill("pdf-extraction") + ... + store.close() + + It also works as a context manager. + + **Server-side only.** A mobile key or a client-side environment ID is + refused in the constructor. + + **Delivery is in the background; retrieval is not.** A daemon thread owns + the connection and fills memory, and ``get_object`` only ever reads what has + already arrived. A process that calls ``get_skill`` immediately after + ``start()`` may see an empty store; ``wait_for_skills`` orders boot against + the first payload. + + **Last known good survives an outage.** A transport failure never empties + the store and never makes ``get_object`` raise, which is what makes + ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and + ``failed`` report the degradation. + + **What arrives is untrusted.** Raw wire objects are held verbatim and + verified at the accessor boundary, not here. In particular an object with no + ``contentHash`` is held and then *withheld*; see + ``StoreDiagnostics.hashless_objects``. + """ + + def __init__( + self, + sdk_key: str, + *, + base_uri: str = DEFAULT_BASE_URI, + mode: Mode = "stream", + poll_interval: float = 30.0, + read_timeout: float | None = None, + initial_backoff: float = 1.0, + max_backoff: float = 30.0, + max_consecutive_failures: int = 10, + _requester: Any = None, + ) -> None: + """ + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches + a live stream in seconds. ``"poll"`` exists for environments that cannot + hold a long-lived connection, and revocation there is one + ``poll_interval`` late. + + *read_timeout* is the only network timeout and bounds every socket + operation of a request, so its meaning and default follow the mode: in + ``"poll"`` it bounds the whole request (``DEFAULT_POLL_TIMEOUT``); in + ``"stream"`` it bounds each wait for the next bytes + (``DEFAULT_STREAM_READ_TIMEOUT``). Must be positive when given. + + *max_backoff* caps every delay between retries, including one the server + asks for with ``Retry-After``. + + *max_consecutive_failures* bounds the retry loop. On exceeding it the + transport stops, logs an error, and the store keeps serving last known + good; ``failed`` reports it. Only failures in a row count: a committed + payload resets the count. + """ + _require_server_side_credential(sdk_key) + if mode not in ("stream", "poll"): + raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') + if poll_interval <= 0: + raise ValueError(f"poll_interval must be positive, got {poll_interval!r}") + if read_timeout is None: + read_timeout = ( + DEFAULT_STREAM_READ_TIMEOUT + if mode == "stream" + else DEFAULT_POLL_TIMEOUT + ) + elif not (math.isfinite(read_timeout) and read_timeout > 0): + raise ValueError(f"read_timeout must be positive, got {read_timeout!r}") + + self._mode: Mode = mode + self._poll_interval = poll_interval + self._initial_backoff = initial_backoff + self._max_backoff = max_backoff + self._max_consecutive_failures = max_consecutive_failures + + self._objects = _SkillObjectSet() + self._reader = _ProtocolReader(self._objects) + self._lock = threading.RLock() + self._listeners: dict[str, list[Callable[[dict[str, Any]], Any]]] = {} + + self._basis: str | None = None + self._etag: str | None = None + + self._requester = _requester or _Requester( + sdk_key.strip(), + base_uri, + read_timeout=read_timeout, + ) + + self._stop = threading.Event() + self._first_payload = threading.Event() + self._thread: threading.Thread | None = None + self._failed_reason: str | None = None + # The open streaming connection, so ``close`` can interrupt its read. + self._connection: Any = None + # Recoverable failures since the last committed payload. Reset at the + # commit rather than when a connection returns: a stream only ever ends + # by being dropped, so resetting on return would count every healthy, + # server-recycled connection as a failure. + self._failures = 0 + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> FDv2SkillStore: + """ + Starts the delivery thread. Idempotent; returns ``self`` so it chains. + + Does not block: use ``wait_for_skills`` when boot ordering matters. + """ + with self._lock: + if self._thread is not None and self._thread.is_alive(): + return self + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="ld-ai-skills-fdv2", daemon=True + ) + self._thread.start() + return self + + def close(self, timeout: float = 5.0) -> None: + """ + Stops delivery. Idempotent, and safe to call from any thread. + + Held content is *not* dropped: a closed store still answers from what it + received. Detaching the store from the accessors is the job of the + package-level ``launchdarkly_ai_server.shutdown()`` coroutine. + """ + self._stop.set() + # The delivery thread is normally blocked in a socket read that no flag + # can reach; without this the join waits out its full timeout. + with self._lock: + connection = self._connection + if connection is not None: + connection.close() + thread = self._thread + if ( + thread is not None + and thread.is_alive() + and thread is not threading.current_thread() + ): + thread.join(timeout=timeout) + + def __enter__(self) -> FDv2SkillStore: + return self.start() + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def wait_for_skills(self, timeout: float = 10.0) -> bool: + """ + Blocks until the first payload has been committed, or *timeout* elapses. + + ``True`` means a payload arrived — not that any skill in it verified, and + not that the environment has any skills. ``diagnostics`` answers the rest. + """ + return self._first_payload.wait(timeout=timeout) + + @property + def failed(self) -> str | None: + """Why delivery stopped for good, or ``None`` while it is running.""" + with self._lock: + return self._failed_reason + + @property + def diagnostics(self) -> StoreDiagnostics: + """A snapshot of what the transport has seen. See ``StoreDiagnostics``.""" + with self._lock: + return StoreDiagnostics(**vars(self._reader.diagnostics)) + + # -- the SkillStore interface ----------------------------------------- + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + if kind != SKILL_OBJECT_KIND: + return None + with self._lock: + return self._objects.get(key, version) + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + if kind != SKILL_OBJECT_KIND: + return {} + with self._lock: + return self._objects.snapshot() + + def add_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Registers *fn* to be called once per changed object, at + ``payload-transferred`` rather than as objects stream in. + + A put notifies with the raw skill object. A revocation notifies with a + ``{"key", "version"}`` tombstone carrying no content, so a listener that + reads content must check for ``content`` rather than assume it. + + *fn* runs on the delivery thread. Keep it cheap and non-blocking. An + exception it raises is logged and swallowed, because a broken listener + must not be able to kill delivery. + """ + with self._lock: + self._listeners.setdefault(kind, []).append(fn) + + def remove_listener(self, kind: str, fn: Callable[[dict[str, Any]], Any]) -> None: + """ + Unregisters *fn* from *kind*. Safe to call from any thread, including + from inside a listener: a removal during one commit takes effect from + the next. + + Removes one occurrence; removing a callable that is not registered is a + no-op, so ``SkillWatcher.close`` can detach unconditionally. + """ + with self._lock: + listeners = self._listeners.get(kind) + if listeners is None: + return + try: + listeners.remove(fn) + except ValueError: + return + + def _notify(self, changes: list[dict[str, Any]]) -> None: + with self._lock: + listeners = list(self._listeners.get(SKILL_OBJECT_KIND, [])) + for raw in changes: + for listener in listeners: + try: + listener(raw) + except Exception: + logger.error( + "A skill store change listener raised; delivery continues", + exc_info=True, + ) + + # -- the delivery loop ------------------------------------------------- + + def _run(self) -> None: + while not self._stop.is_set(): + try: + if self._mode == "stream": + self._stream_once() + else: + self._poll_once() + # A poll that returned is a current answer even when it committed + # nothing (HTTP 304). A stream never returns normally; its + # successes are counted at each commit in ``_apply``. + self._record_success() + except _FatalTransportError as exc: + self._give_up(str(exc)) + return + except _RecoverableTransportError as exc: + with self._lock: + self._failures += 1 + failures = self._failures + self._reader.diagnostics.connection_failures = failures + self._reader.diagnostics.last_error = str(exc) + if failures > self._max_consecutive_failures: + self._give_up( + f"gave up after {failures} consecutive failures; " + f"last error: {exc}" + ) + return + delay = exc.retry_after + if delay is None or not math.isfinite(delay): + delay = _backoff_delay( + failures, base=self._initial_backoff, maximum=self._max_backoff + ) + # ``Retry-After`` is a request and ``max_backoff`` is a promise. + # The header may come from a proxy rather than LaunchDarkly, and + # a value in the hours would park revocation for that long. + delay = min(delay, self._max_backoff) + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) + if self._stop.wait(delay): + return + continue + except Exception as exc: # pragma: no cover - defensive + self._give_up(f"unexpected error in skill delivery: {exc!r}") + logger.error("Unexpected error in skill delivery", exc_info=True) + return + + if self._mode == "poll" and self._stop.wait(self._poll_interval): + return + + def _record_success(self) -> None: + with self._lock: + self._failures = 0 + self._reader.diagnostics.connection_failures = 0 + + def _give_up(self, reason: str) -> None: + with self._lock: + self._failed_reason = reason + self._reader.diagnostics.last_error = reason + logger.error( + "Skill delivery has stopped and will not retry: %s. The store keeps " + "serving the last content it received; skills will not update until " + "the process restarts with a working connection.", + reason, + ) + # Unblock anyone waiting on a first payload that is never coming. + self._first_payload.set() + + def _apply(self, name: str, data: Any) -> None: + """ + Feeds one event to the reader, publishes a commit, and raises the + transport error the event calls for, if any. + """ + with self._lock: + outcome = self._reader.handle(name, data) + if outcome.committed and outcome.basis is not None: + self._basis = outcome.basis + if outcome.committed: + # A commit breaks the row of consecutive failures. + self._record_success() + self._first_payload.set() + if outcome.changes: + self._notify(outcome.changes) + if outcome.fatal: + raise _FatalTransportError(outcome.fatal) + if outcome.disconnect: + raise _RecoverableTransportError(outcome.disconnect) + + def _poll_once(self) -> None: + with self._lock: + basis, etag = self._basis, self._etag + result = self._requester.poll(basis, etag) + with self._lock: + self._etag = result.etag + if result.not_modified: + logger.debug("Skill payload unchanged (HTTP 304)") + # A 304 counts as a first payload, so a boot that reconnects with a + # cached basis is not blocked on a transfer the server will not send. + self._first_payload.set() + return + for name, data in result.events: + self._apply(name, data) + + def _stream_once(self) -> None: + with self._lock: + basis = self._basis + connection = self._requester.stream(basis) + with self._lock: + self._connection = connection + try: + # ``close`` may have run while the connect was in flight and found + # no connection to interrupt; this is the last chance to notice + # before the read below blocks. + if self._stop.is_set(): + return + for name, data in connection.events: + if self._stop.is_set(): + return + self._apply(name, data) + except Exception: + if self._stop.is_set(): + # ``close`` interrupted the read on purpose. + return + raise + finally: + connection.close() + with self._lock: + self._connection = None + # A stream that ends without a goodbye is a dropped connection. + raise _RecoverableTransportError("the FDv2 stream closed unexpectedly") diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 05263057..e421df6a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -267,7 +267,7 @@ async def watch_skills( "watch_skills needs a skill store that implements add_listener(kind, " "fn); the configured store does not, so delivery changes cannot be " "observed. Use write_skills for a one-shot reconcile, or configure a " - "store with a delivery transport." + "store with a delivery transport (FDv2SkillStore)." ) if debounce < 0: raise ValueError(f"debounce must not be negative, got {debounce!r}") diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index d3f83095..a07ea6ae 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -603,6 +603,18 @@ async def test_all_skills_raises_actionably_when_no_store(self) -> None: with pytest.raises(RuntimeError, match="skill store"): await all_skills() + async def test_the_no_store_message_names_the_delivery_store_first(self) -> None: + """ + A deployment that hits this message must be pointed at the store that + receives content from LaunchDarkly, not only at the development one. + """ + with pytest.raises(RuntimeError) as reported: + await get_skill("a") + message = str(reported.value) + assert "FDv2SkillStore" in message + assert "InMemorySkillStore" in message + assert message.index("FDv2SkillStore") < message.index("InMemorySkillStore") + async def test_shutdown_clears_the_store( self, make_raw_skill: Any, mock_ld_client: Any ) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index b88d14d9..0cce2f0c 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1,26 +1,55 @@ """ -Tests for the FDv2 skill delivery protocol. - -Wire semantics — which objects are skills, the skill's version in the wire ``key`` -versus the payload's in ``version``, revocation, mixed payloads, the commit at ``payload-transferred`` — are asserted -against ``_ProtocolReader``, which has no I/O, so each case reads as the contract -it is rather than as a server script. +Tests for the FDv2 skill delivery transport. + +Two layers, deliberately: + +- **A real fake endpoint.** ``_FakeFDv2Endpoint`` is an in-process + ``ThreadingHTTPServer`` that implements the wire contract — the ``basis`` + query parameter, ``Authorization``, ``If-None-Match``/304, the + ``{"events": [...]}`` polling envelope, and SSE for streaming. The store under + test opens real sockets against it, so request construction and header + handling are exercised rather than mocked. +- **The protocol reader driven directly.** Wire semantics — which objects are + skills, the skill's version in the wire ``key`` versus the payload's in + ``version``, revocation, mixed payloads — are + asserted against ``_ProtocolReader``, which has no I/O, so those cases read as + the contract they are instead of as a server script. """ from __future__ import annotations import hashlib +import json +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar +from urllib.parse import parse_qs, urlparse import pytest -from launchdarkly_ai_server import InMemorySkillStore +from launchdarkly_ai_server import ( + FDv2SkillStore, + InMemorySkillStore, + all_skills, + get_skill, + get_skill_result, + init_client, + watch_skills, +) from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND from launchdarkly_ai_server.skills_fdv2 import ( + DEFAULT_POLL_TIMEOUT, + DEFAULT_STREAM_READ_TIMEOUT, FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, + _backoff_delay, _is_skill_event, _ProtocolReader, + _RecoverableTransportError, + _Requester, + _retry_after_seconds, _SkillObjectSet, _store_object_from_put, _tombstone_from_delete, @@ -28,6 +57,7 @@ pytestmark = pytest.mark.usefixtures("reset_skill_state") +SDK_KEY = "sdk-00000000-0000-4000-8000-000000000000" SKILL_BODY = "---\nname: PDF Extraction\n---\nExtract text from PDFs.\n" @@ -143,6 +173,182 @@ def full_payload( ) +# --------------------------------------------------------------------------- +# The fake endpoint +# --------------------------------------------------------------------------- + + +class _FakeFDv2Endpoint: + """ + An in-process server implementing the SDK-facing FDv2 contract. + + Scripted per request: ``queue_poll`` appends a response for the next + ``/sdk/poll``, ``queue_stream`` appends a sequence of SSE events for the next + ``/sdk/stream``. Every request's method, path, query and headers are recorded + in ``requests`` so the tests can assert on what the store actually sent — + which is the only way ``basis`` round-tripping and ``If-None-Match`` can be + checked at all. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self._polls: list[dict[str, Any]] = [] + self._streams: list[list[dict[str, Any]]] = [] + self._lock = threading.Lock() + self.hold_stream_open = False + self._release = threading.Event() + + endpoint = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args: Any) -> None: + return + + def do_GET(self) -> None: + parsed = urlparse(self.path) + query = {k: v[0] for k, v in parse_qs(parsed.query).items()} + with endpoint._lock: + endpoint.requests.append( + { + "path": parsed.path, + "query": query, + "authorization": self.headers.get("Authorization"), + "if_none_match": self.headers.get("If-None-Match"), + "accept": self.headers.get("Accept"), + } + ) + if parsed.path == "/sdk/poll": + endpoint._serve_poll(self) + elif parsed.path == "/sdk/stream": + endpoint._serve_stream(self) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + class Server(ThreadingHTTPServer): + # Handler threads are not joined on shutdown: a test that ends while + # a stream is deliberately held open should not pay for the hold. + daemon_threads = True + + self._server = Server(("127.0.0.1", 0), Handler) + # A short poll interval so `shutdown` is prompt: the default 0.5s is + # paid at the teardown of every test that touches the endpoint. + self._thread = threading.Thread( + target=lambda: self._server.serve_forever(poll_interval=0.01), daemon=True + ) + self._thread.start() + + @property + def base_uri(self) -> str: + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + # -- scripting --------------------------------------------------------- + + def queue_poll( + self, + payload_events: list[dict[str, Any]] | None = None, + *, + status: int = 200, + etag: str | None = None, + retry_after: str | None = None, + ) -> None: + with self._lock: + self._polls.append( + { + "status": status, + "events": payload_events or [], + "etag": etag, + "retry_after": retry_after, + } + ) + + def queue_stream(self, payload_events: list[dict[str, Any]]) -> None: + with self._lock: + self._streams.append(payload_events) + + # -- serving ----------------------------------------------------------- + + def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + response = ( + self._polls.pop(0) if self._polls else {"status": 304, "events": []} + ) + status = response["status"] + handler.send_response(status) + if response.get("etag"): + handler.send_header("ETag", response["etag"]) + if response.get("retry_after"): + handler.send_header("Retry-After", response["retry_after"]) + if status in (200,): + body = json.dumps({"events": response["events"]}).encode("utf-8") + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + handler.send_header("Content-Length", "0") + handler.end_headers() + + def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + payload_events = self._streams.pop(0) if self._streams else [] + handler.send_response(200) + handler.send_header("Content-Type", "text/event-stream") + handler.send_header("Cache-Control", "no-cache") + handler.send_header("Transfer-Encoding", "chunked") + handler.end_headers() + for event in payload_events: + chunk = ( + f"event: {event['event']}\ndata: {json.dumps(event.get('data'))}\n\n" + ).encode() + handler.wfile.write(f"{len(chunk):X}\r\n".encode() + chunk + b"\r\n") + handler.wfile.flush() + if self.hold_stream_open: + # Keeps the connection up so a test can assert on the store's state + # without racing the reconnect path. Released on ``close`` so the + # hold costs the suite nothing once the test is done with it. + self._release.wait(timeout=10) + handler.wfile.write(b"0\r\n\r\n") + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _FakeFDv2Endpoint() + yield server + server.close() + + +def poll_store(endpoint: Any, **kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="poll", + poll_interval=kwargs.pop("poll_interval", 0.05), + initial_backoff=kwargs.pop("initial_backoff", 0.01), + max_backoff=kwargs.pop("max_backoff", 0.05), + read_timeout=kwargs.pop("read_timeout", 5.0), + **kwargs, + ) + + +def wait_until(predicate: Any, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.01) + return predicate() + + # --------------------------------------------------------------------------- # Identifying skill objects, and ignoring everything else # --------------------------------------------------------------------------- @@ -827,6 +1033,608 @@ def test_snapshot_agrees(self) -> None: assert memory.all_objects(SKILL_OBJECT_KIND) == objects.snapshot() +# --------------------------------------------------------------------------- +# The store against the fake endpoint +# --------------------------------------------------------------------------- + + +class TestPollingAgainstTheEndpoint: + def test_a_polled_skill_becomes_retrievable_through_the_accessors( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert raw["version"] == 3 + + def test_the_request_carries_the_sdk_key_and_no_data_model_version( + self, endpoint: Any + ) -> None: + """ + No ``mv``: that parameter selects the *flag* data model, the connection + rejects any value but the flag default, and the generic agent-skill + payload is served regardless of it. Sending ``mv=1`` — the skill + payload's own model version — gets the whole connection refused. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + first = endpoint.requests[0] + assert first["path"] == "/sdk/poll" + assert first["authorization"] == SDK_KEY + assert "mv" not in first["query"] + + def test_the_first_request_sends_no_basis(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert "basis" not in endpoint.requests[0]["query"] + + def test_the_basis_from_payload_transferred_is_echoed_on_the_next_request( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload(("put-object", put_skill()), state="selector-abc") + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["query"]["basis"] == "selector-abc" + + def test_the_basis_advances_across_successive_payloads(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()), state="basis-1")) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint): + assert wait_until(lambda: len(endpoint.requests) >= 3) + bases = [r["query"].get("basis") for r in endpoint.requests[:3]] + assert bases == [None, "basis-1", "basis-2"] + + def test_an_etag_is_returned_as_if_none_match(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert endpoint.requests[1]["if_none_match"] == 'W/"v1"' + + def test_a_304_keeps_the_held_content(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill())), etag='W/"v1"') + endpoint.queue_poll(status=304) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert wait_until(lambda: len(endpoint.requests) >= 3) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.payloads_transferred == 1 + assert store.failed is None + + def test_a_304_before_any_payload_still_releases_wait_for_skills( + self, endpoint: Any + ) -> None: + """A reconnect with a cached basis has nothing to transfer; boot must not + block on a payload the server has no reason to send.""" + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + + def test_a_mixed_payload_over_the_wire_yields_only_the_skill( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_flag("flag-a")), + ("put-object", put_segment("beta")), + ("put-object", put_skill("pdf-extraction")), + ("put-object", put_flag("flag-b")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + held = store.all_objects(SKILL_OBJECT_KIND) + assert len(held) == 1 + assert next(iter(held.values()))["key"] == "pdf-extraction" + assert store.diagnostics.objects_ignored == 3 + + def test_a_revocation_over_the_wire_removes_the_skill(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + + def test_the_store_asks_for_only_the_kind_it_serves(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.get_object("flag", "pdf-extraction") is None + assert store.all_objects("flag") == {} + + +class TestStreamingAgainstTheEndpoint: + def test_a_streamed_payload_lands(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_the_stream_request_advertises_event_stream(self, endpoint: Any) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + store.wait_for_skills(timeout=5) + finally: + store.close() + assert endpoint.requests[0]["path"] == "/sdk/stream" + assert endpoint.requests[0]["accept"] == "text/event-stream" + + def test_a_streamed_revocation_arrives_without_a_restart( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + try: + store.start() + assert wait_until( + lambda: ( + store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + and store.diagnostics.objects_revoked == 1 + ) + ) + finally: + store.close() + + def test_a_dropped_stream_reconnects_with_the_basis_it_reached( + self, endpoint: Any + ) -> None: + endpoint.queue_stream( + full_payload(("put-object", put_skill()), state="basis-1") + ) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.05, + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + finally: + store.close() + assert endpoint.requests[1]["query"]["basis"] == "basis-1" + + def test_close_returns_promptly_while_a_stream_is_open(self, endpoint: Any) -> None: + """ + The delivery thread is blocked in a socket read that no stop flag can + reach, so ``close`` closes the connection under it. Without that, every + shutdown of a healthy stream waits out the join timeout. + """ + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 1.0 + + def test_an_interrupted_stream_is_not_reported_as_a_failure( + self, endpoint: Any + ) -> None: + endpoint.hold_stream_open = True + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + store = FDv2SkillStore(SDK_KEY, base_uri=endpoint.base_uri, mode="stream") + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.failed is None + + def test_content_survives_a_reconnect(self, endpoint: Any) -> None: + endpoint.queue_stream(full_payload(("put-object", put_skill()))) + endpoint.queue_stream(events(("heart-beat", None))) + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + try: + store.start() + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + + +class _ScriptedConnection: + """Stands in for ``_StreamConnection``: an event iterator plus a close.""" + + def __init__(self, payload_events: Any) -> None: + self.events = iter(payload_events) + self.closed = False + + def close(self) -> None: + self.closed = True + + +class _ScriptedRequester: + """Raises a scripted sequence, so backoff is asserted without real sockets.""" + + def __init__(self, *outcomes: Any) -> None: + self.outcomes = list(outcomes) + self.calls: list[tuple[str | None, str | None]] = [] + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.calls.append((basis, etag)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return outcome + + def stream(self, basis: str | None) -> Any: + self.calls.append((basis, None)) + outcome = ( + self.outcomes.pop(0) if self.outcomes else _RecoverableTransportError("x") + ) + if isinstance(outcome, Exception): + raise outcome + return _ScriptedConnection(outcome) + + +class _RecyclingRequester: + """ + A healthy server that recycles connections: every ``stream`` call succeeds, + transfers a full payload, and then ends the connection, as LaunchDarkly and + any proxy in between do to a long-lived stream. + """ + + def __init__(self) -> None: + self.connections = 0 + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _ScriptedConnection( + [ + (e["event"], e["data"]) + for e in full_payload( + ("put-object", put_skill()), state=f"basis-{self.connections}" + ) + ] + ) + + +class _BlockingConnection: + """A stream that never produces an event until it is closed.""" + + def __init__(self) -> None: + self._closed = threading.Event() + + @property + def events(self) -> Any: + self._closed.wait() + return iter(()) + + def close(self) -> None: + self._closed.set() + + +class _SlowConnectRequester: + """ + A ``stream`` whose connect does not return until the test releases it, + standing in for a slow TLS handshake, followed by a read that never yields. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def stream(self, basis: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + return _BlockingConnection() + + +def stream_store(**kwargs: Any) -> FDv2SkillStore: + return FDv2SkillStore( + SDK_KEY, + mode="stream", + initial_backoff=kwargs.pop("initial_backoff", 0.001), + max_backoff=kwargs.pop("max_backoff", 0.002), + **kwargs, + ) + + +class TestFailureHandling: + def test_a_403_stops_delivery_and_explains_why( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(status=403) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "403" in store.failed + assert "opt-in" in store.failed + assert any("opt-in" in r.getMessage() for r in caplog.records) + + def test_a_401_stops_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is not None + + def test_a_fatal_failure_keeps_last_known_good_servable( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=403) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_500_is_retried(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=503) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_a_retry_resets_the_failure_count_on_success(self, endpoint: Any) -> None: + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + + def test_retries_are_bounded(self) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + max_backoff=0.002, + max_consecutive_failures=3, + _requester=_ScriptedRequester(), + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + # Four, not three: the bound is the number of failures *tolerated*, + # so the run that exceeds it is the one that gives up. + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_recycled_stream_connections_are_not_failures(self) -> None: + # A streaming connection only ever ends by being dropped, so a loop + # that counted every drop as a failure would give up on a healthy + # server after max_consecutive_failures + 1 recycles, and delivery + # (including revocation) would silently stop for the process lifetime. + requester = _RecyclingRequester() + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + assert store.diagnostics.payloads_transferred >= 8 + # A drop is a failure until the next commit clears it, so the count + # may read 1 mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + finally: + store.close() + + def test_a_stream_commit_resets_the_failure_count(self) -> None: + payload = [ + (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) + ] + requester = _ScriptedRequester( + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + _RecoverableTransportError("x"), + payload, + ) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) + # Three failures reach the bound, then a commit, then the exhausted + # requester fails on every reconnect. The count must start again at + # the commit: the stream's own drop is failure one, and three more + # connects are owed before giving up. Carrying the three over would + # give up on the drop itself, with no further connect at all. + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + assert "last error: x" in store.failed + assert len(requester.calls) == 7 + finally: + store.close() + + def test_stream_retries_are_bounded(self) -> None: + store = stream_store( + max_consecutive_failures=3, _requester=_ScriptedRequester() + ) + try: + store.start() + assert wait_until(lambda: store.failed is not None) + assert "gave up after 4 consecutive failures" in store.failed + finally: + store.close() + + def test_a_retry_after_header_is_honoured(self) -> None: + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=0.25), + ) + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=10.0, + initial_backoff=5.0, + _requester=requester, + ) + try: + started = time.monotonic() + store.start() + assert wait_until(lambda: len(requester.calls) >= 2, timeout=3) + elapsed = time.monotonic() - started + # The server asked for 0.25s; our own backoff would have been 5s. + assert 0.2 <= elapsed < 3.0 + finally: + store.close() + + def test_a_retry_after_header_is_parsed_off_the_wire(self, endpoint: Any) -> None: + endpoint.queue_poll(status=429, retry_after="0") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint, initial_backoff=5.0) as store: + # If Retry-After were ignored the 5s backoff would blow the timeout. + assert store.wait_for_skills(timeout=3) is True + + @pytest.mark.parametrize("raw", ["inf", "Infinity", "-inf", "nan", "1e309"]) + def test_a_non_finite_retry_after_is_ignored(self, raw: str) -> None: + assert _retry_after_seconds({"Retry-After": raw}) is None + + def test_retry_after_parsing_keeps_its_edges(self) -> None: + assert _retry_after_seconds({"Retry-After": "0"}) == 0.0 + assert _retry_after_seconds({"Retry-After": "-5"}) == 0.0 + assert ( + _retry_after_seconds({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) + is None + ) + assert _retry_after_seconds({"Retry-After": "2.5"}) == 2.5 + + @pytest.mark.parametrize("retry_after", [float("inf"), float("nan"), 86400.0]) + def test_an_unreasonable_retry_after_neither_kills_delivery_nor_parks_it( + self, retry_after: float + ) -> None: + # An infinite wait would overflow inside the retry handler and kill the + # thread with `failed` still None; a day-long one would be honoured to + # the second. Both must fall back to the max_backoff cap and carry on. + requester = _ScriptedRequester( + _RecoverableTransportError("slow down", retry_after=retry_after), + [ + (e["event"], e["data"]) + for e in full_payload(("put-object", put_skill())) + ], + ) + store = stream_store(max_backoff=0.05, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + assert store._thread is not None and store._thread.is_alive() + finally: + store.close() + + def test_a_non_finite_retry_after_off_the_wire_falls_back_to_backoff( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(status=429, retry_after="inf") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=3) is True + assert store.failed is None + + def test_backoff_is_exponential_and_capped(self) -> None: + assert _backoff_delay(1, base=1.0, maximum=30.0, jitter=0.0) == 1.0 + assert _backoff_delay(2, base=1.0, maximum=30.0, jitter=0.0) == 2.0 + assert _backoff_delay(3, base=1.0, maximum=30.0, jitter=0.0) == 4.0 + assert _backoff_delay(20, base=1.0, maximum=30.0, jitter=0.0) == 30.0 + + def test_jitter_never_exceeds_the_cap(self) -> None: + for attempt in range(1, 12): + for _ in range(50): + assert 0.0 <= _backoff_delay(attempt, base=1.0, maximum=5.0) <= 5.0 + + def test_a_malformed_polling_envelope_is_recoverable_not_fatal( + self, endpoint: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + mode="poll", + poll_interval=0.01, + initial_backoff=0.001, + _requester=_ScriptedRequester( + _RecoverableTransportError("polling response had no 'events' array") + ), + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + finally: + store.close() + + def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill("first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + with poll_store(endpoint) as store: + store.add_listener(SKILL_OBJECT_KIND, lambda _raw: 1 / 0) + assert wait_until( + lambda: store.get_object(SKILL_OBJECT_KIND, "second") is not None + ) + assert store.failed is None + + # --------------------------------------------------------------------------- # The contentHash gap # --------------------------------------------------------------------------- @@ -854,6 +1662,63 @@ class TestMissingContentHash: itself and verify nothing. """ + async def test_a_hashless_skill_is_withheld_with_the_right_reason( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + outcome = await get_skill_result("pdf-extraction") + assert outcome.skill is None + assert outcome.reason == "integrity_failure" + assert await get_skill("pdf-extraction") is None + assert await all_skills() == [] + + async def test_the_object_is_still_held_so_the_outcome_is_not_absent( + self, endpoint: Any + ) -> None: + """ + Holding it is what makes the failure diagnosable. Dropping it at the + transport would report ``absent`` — indistinguishable from "no such + skill" — and would additionally let a prune delete the last known-good + copy already on disk. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + raw = store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") + assert raw is not None + assert "contentHash" not in raw + await init_client(options={"skillStore": store}, client=object()) + assert (await get_skill_result("pdf-extraction")).reason != "absent" + + def test_the_store_counts_hashless_objects(self, endpoint: Any) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ("put-object", put_skill("c")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 2 + assert store.diagnostics.skill_objects_received == 3 + + def test_a_hashless_object_logs_an_error_naming_the_reason_code( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(omit_hash=True)))) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "missing_content_hash" in rendered + assert "pdf-extraction" in rendered + assert "contentHash" in rendered + def test_a_redelivered_hashless_object_logs_once_per_store( self, caplog: Any ) -> None: @@ -895,3 +1760,456 @@ def test_two_live_stores_do_not_suppress_each_other(self, caplog: Any) -> None: drive(one, payload) drive(two, payload) assert len(_per_object_hashless_errors(caplog)) == 2 + + def test_a_wholly_hashless_payload_says_so_once( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b", omit_hash=True)), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + summaries = [ + r + for r in caplog.records + if "No skill content will resolve" in r.getMessage() + ] + assert len(summaries) == 1 + assert "All 2 skill object(s)" in summaries[0].getMessage() + + def test_a_partly_hashed_payload_does_not_claim_total_failure( + self, endpoint: Any, caplog: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill("a", omit_hash=True)), + ("put-object", put_skill("b")), + ) + ) + with caplog.at_level("ERROR"): + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + rendered = "\n".join(r.getMessage() for r in caplog.records) + assert "No skill content will resolve" not in rendered + + async def test_a_hash_that_does_not_match_is_a_different_failure( + self, endpoint: Any + ) -> None: + """``missing_content_hash`` and ``hash_mismatch`` must not collapse: one + means the envelope carried no hash, the other means the content did not + match the hash it carried.""" + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(content_hash=_hash("something else"))) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + assert store.diagnostics.hashless_objects == 0 + await init_client(options={"skillStore": store}, client=object()) + assert ( + await get_skill_result("pdf-extraction") + ).reason == "integrity_failure" + + async def test_a_hashed_skill_resolves_end_to_end(self, endpoint: Any) -> None: + """The positive control: a well-formed envelope resolves end to end.""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + skill = await get_skill("pdf-extraction") + assert skill is not None + assert skill.key == "pdf-extraction" + assert skill.version == 3 + assert skill.content == SKILL_BODY.encode("utf-8") + assert skill.content_hash == _hash(SKILL_BODY) + assert skill.name == "PDF Extraction" + + async def test_a_pinned_reference_resolves_to_the_pinned_object_version( + self, endpoint: Any + ) -> None: + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=2, content="v2 body")), + ("put-object", put_skill(object_version=5, content="v5 body")), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + pinned = await get_skill("pdf-extraction", version=2) + assert pinned is not None + assert pinned.content == b"v2 body" + newest = await get_skill("pdf-extraction") + assert newest is not None + assert newest.version == 5 + + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( + self, endpoint: Any + ) -> None: + """ + The end-to-end form of the wire-key/``version`` assertion. + + Asking for the payload version resolves nothing — reported ``absent``, + because the store answers "I hold no such version" rather than answering + with the wrong one. The version that *does* resolve is the one after the + delimiter in the object's wire ``key``. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3, payload_version=42)) + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + by_payload_version = await get_skill_result("pdf-extraction", version=42) + assert by_payload_version.skill is None + assert by_payload_version.reason == "absent" + assert await get_skill("pdf-extraction", version=3) is not None + + +# --------------------------------------------------------------------------- +# Server-side only +# --------------------------------------------------------------------------- + + +class TestServerSideOnly: + def test_a_mobile_key_is_refused(self) -> None: + with pytest.raises(ValueError, match="mobile key"): + FDv2SkillStore("mob-00000000-0000-4000-8000-000000000000") + + def test_a_client_side_environment_id_is_refused(self) -> None: + with pytest.raises(ValueError, match="client-side"): + FDv2SkillStore("0123456789abcdef01234567") + + def test_an_empty_credential_is_refused(self) -> None: + with pytest.raises(ValueError, match="server-side SDK key"): + FDv2SkillStore(" ") + + def test_a_server_side_key_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY) is not None + + def test_an_unrecognised_credential_shape_warns_but_is_allowed( + self, caplog: Any + ) -> None: + """Private instances and test doubles issue keys without the public prefix.""" + with caplog.at_level("WARNING"): + FDv2SkillStore("my-private-instance-credential") + assert any("server-side SDK key" in r.message for r in caplog.records) + + def test_an_unknown_mode_is_refused(self) -> None: + with pytest.raises(ValueError, match="stream"): + FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# The eager re-reconcile, end to end over the transport +# --------------------------------------------------------------------------- + + +class TestWatchSkillsOverTheTransport: + """ + ``watch_skills`` against a live ``FDv2SkillStore``. The watcher's own + behaviour — debounce, refusal of a store without ``add_listener``, detaching + on close — is covered in ``test_skills_watch.py`` against the in-memory + store; these are the cases that only mean something with a transport + underneath: a wire-level revocation, a new skill version, and an outage. + """ + + async def test_a_revocation_prunes_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + """ + The store's change listener drives the reconcile, so the file goes away + seconds after the ``delete-object`` rather than at the next process start. + """ + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-changes")), + ("delete-object", delete_skill()), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + report, watcher = await watch_skills( + "*", tmp_path / "skills", debounce=0.05 + ) + try: + written = tmp_path / "skills" / "pdf-extraction" / "SKILL.md" + assert written.exists() + assert any(a.action == "written" for a in report.actions) + assert wait_until(lambda: not written.exists(), timeout=10) + finally: + watcher.close() + + async def test_a_new_version_is_rewritten_without_a_restart( + self, endpoint: Any, tmp_path: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill(content="first")))) + endpoint.queue_poll( + events( + ("server-intent", server_intent("xfer-full")), + ("put-object", put_skill(object_version=4, content="second")), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.queue_poll(status=304) + + with poll_store(endpoint, poll_interval=0.2) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.read_text() == "first" + assert wait_until(lambda: written.read_text() == "second", timeout=10) + finally: + watcher.close() + + async def test_the_default_keeps_last_known_good_during_an_outage( + self, endpoint: Any, tmp_path: Any + ) -> None: + """``on_unavailable="keep"`` is the default: an outage must not read as + "everything was revoked".""" + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + endpoint.queue_poll(status=500) + with poll_store(endpoint, poll_interval=0.05) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + _report, watcher = await watch_skills("*", tmp_path / "s", debounce=0.05) + try: + written = tmp_path / "s" / "pdf-extraction" / "SKILL.md" + assert written.exists() + # ``last_error`` rather than ``connection_failures``: the counter + # resets on the next successful poll, so asserting on it races + # the retry that is supposed to happen. + assert wait_until( + lambda: store.diagnostics.last_error is not None, timeout=10 + ) + time.sleep(0.3) + assert written.exists() + finally: + watcher.close() + + +# --------------------------------------------------------------------------- +# Listener registration +# --------------------------------------------------------------------------- + + +class TestListenerRegistration: + @staticmethod + def _skill_listeners(store: Any) -> list[Any]: + return list(store._listeners.get(SKILL_OBJECT_KIND, [])) + + def test_fdv2_remove_listener_of_an_unregistered_callable_is_a_no_op( + self, endpoint: Any + ) -> None: + with poll_store(endpoint) as store: + store.remove_listener(SKILL_OBJECT_KIND, print) + store.add_listener(SKILL_OBJECT_KIND, print) + store.remove_listener("flag", print) + store.remove_listener(SKILL_OBJECT_KIND, print) + store.remove_listener(SKILL_OBJECT_KIND, print) + assert self._skill_listeners(store) == [] + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestLifecycle: + def test_start_is_idempotent(self, endpoint: Any) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + try: + assert store.start() is store + assert store.start() is store + assert store.wait_for_skills(timeout=5) + finally: + store.close() + + def test_close_is_idempotent(self, endpoint: Any) -> None: + store = poll_store(endpoint) + store.start() + store.close() + store.close() + + def test_close_during_a_slow_connect_returns_promptly(self) -> None: + # Before the connect returns there is no connection for close() to + # interrupt. If the delivery thread then enters the read anyway, close() + # sits out its whole join timeout on a stream that will never speak. + requester = _SlowConnectRequester() + store = stream_store(_requester=requester) + store.start() + assert requester.entered.wait(timeout=5) + threading.Timer(0.1, requester.release.set).start() + started = time.monotonic() + store.close(timeout=5.0) + elapsed = time.monotonic() - started + assert elapsed < 2.0 + assert store._thread is not None + assert not store._thread.is_alive() + + def test_a_closed_store_still_answers_from_what_it_received( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store = poll_store(endpoint) + store.start() + store.wait_for_skills(timeout=5) + store.close() + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + + def test_wait_for_skills_times_out_rather_than_hanging(self) -> None: + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=60, _requester=_ScriptedRequester() + ) + try: + assert store.wait_for_skills(timeout=0.05) is False + finally: + store.close() + + def test_the_store_satisfies_the_seam_before_it_starts(self) -> None: + store = FDv2SkillStore(SDK_KEY) + assert store.get_object(SKILL_OBJECT_KIND, "anything") is None + assert store.all_objects(SKILL_OBJECT_KIND) == {} + + +# --------------------------------------------------------------------------- +# Timeouts +# --------------------------------------------------------------------------- + + +class _BlackHole: + """ + A listening socket that accepts connections and never sends a byte. + + This is the host ``read_timeout`` exists for: the TCP handshake completes, so + nothing fails fast, and then no response ever comes. A request against it can + only end by timing out, which makes the elapsed time a direct measurement of + the timeout actually applied. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._accept_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _accept_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def black_hole() -> Any: + server = _BlackHole() + yield server + server.close() + + +class TestTimeouts: + """ + ``read_timeout`` is the only network timeout, and every request honours it. + + The bounds asserted here are loose on purpose: the point is that a request + against an unresponsive host fails in roughly ``read_timeout`` rather than in + minutes, and that a regression back to a much longer default fails this + suite quickly instead of hanging it. + """ + + def test_a_poll_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError) as excinfo: + requester.poll(None, None) + elapsed = time.monotonic() - started + assert 0.2 <= elapsed < 2.0 + assert "timed out" in str(excinfo.value) + + def test_a_stream_against_an_unresponsive_host_fails_within_read_timeout( + self, black_hole: Any + ) -> None: + requester = _Requester(SDK_KEY, black_hole.base_uri, read_timeout=0.3) + started = time.monotonic() + with pytest.raises(_RecoverableTransportError): + requester.stream(None) + assert time.monotonic() - started < 2.0 + + def test_the_store_reports_the_timeout_and_keeps_going( + self, black_hole: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=black_hole.base_uri, + mode="poll", + poll_interval=0.05, + initial_backoff=0.01, + max_backoff=0.05, + read_timeout=0.3, + ) + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 1) + assert store.failed is None + assert "timed out" in (store.diagnostics.last_error or "") + finally: + store.close() + + def test_the_default_bound_depends_on_the_mode(self) -> None: + assert DEFAULT_POLL_TIMEOUT == 10.0 + assert DEFAULT_STREAM_READ_TIMEOUT == 300.0 + polling = FDv2SkillStore(SDK_KEY, mode="poll") + streaming = FDv2SkillStore(SDK_KEY, mode="stream") + assert polling._requester._read_timeout == DEFAULT_POLL_TIMEOUT + assert streaming._requester._read_timeout == DEFAULT_STREAM_READ_TIMEOUT + + @pytest.mark.parametrize("mode", ["poll", "stream"]) + def test_an_explicit_read_timeout_overrides_the_default(self, mode: Any) -> None: + store = FDv2SkillStore(SDK_KEY, mode=mode, read_timeout=42.0) + assert store._requester._read_timeout == 42.0 + + @pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) + def test_a_non_positive_read_timeout_is_rejected(self, value: float) -> None: + with pytest.raises(ValueError, match="read_timeout"): + FDv2SkillStore(SDK_KEY, read_timeout=value) + + def test_there_is_no_separate_connect_timeout(self) -> None: + # ``urllib`` cannot bound the connect separately from the reads, so the + # constructor does not offer a parameter that would only pretend to. + with pytest.raises(TypeError): + FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] diff --git a/packages/client/tests/test_skills_watch.py b/packages/client/tests/test_skills_watch.py index 48a95f07..b44ffea0 100644 --- a/packages/client/tests/test_skills_watch.py +++ b/packages/client/tests/test_skills_watch.py @@ -4,7 +4,10 @@ The watcher is wired to the ``SkillStore`` interface, not to any one transport: it needs a store that implements ``add_listener``, and nothing more. These tests therefore drive it from ``InMemorySkillStore``, whose ``put`` notifies its -listeners synchronously, and from small hand-written store doubles. +listeners synchronously, and from small hand-written store doubles. The +end-to-end path — a ``delete-object`` arriving over a live FDv2 connection and +pruning a skill's files — is exercised in ``test_skills_fdv2.py``, where the fake +endpoint lives. Every test writes only inside pytest's ``tmp_path``. The watcher runs a real worker thread, so tests wait on observable outcomes rather than on fixed sleeps From c285ff533991b3e74a2d0b976a91de1008302596 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 14:04:26 -0400 Subject: [PATCH 24/43] fix(client): retry an FDv2 stream that dies mid-read _Requester.stream wrapped only the connect as recoverable, so a read timeout, reset or truncated chunk in the body reached the delivery loop as whatever the socket raised. The loop read that as a bug and gave up: delivery stopped for the process lifetime, taking updates and revocations with it, the first time a socket died. read_timeout exists to bound a stream that has gone quiet so the loop can reconnect, and tripping it did the opposite. The body now carries the same promise the connect already did. Wrapping the line source rather than the whole read keeps protocol reader errors out of it: those are raised from the consumer's loop body, where they still surface as the bugs they are. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 21 ++++++- packages/client/tests/test_skills_fdv2.py | 61 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index da84a3b4..18b81842 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1032,6 +1032,25 @@ def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: return events +def _iter_stream_lines(response: Any) -> Any: + """ + Yields a streaming body's raw lines, presenting a read failure as retryable. + + A live stream dies mid-body far more often than it refuses to open: a read + timeout on a stream that went quiet, a reset, a truncated chunk. Each of + those arrives as whatever the socket raised, and the delivery loop retries + only the transport errors this module defines — anything else it reads as a + bug and stops for the process lifetime. Connecting is already wrapped in + ``_Requester.stream``; this is the same promise for the body. + """ + try: + yield from response + except Exception as exc: + raise _RecoverableTransportError( + f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" + ) from exc + + def _iter_sse(response: Any) -> Any: """ Decodes an SSE body into ``(event name, data)`` pairs. @@ -1042,7 +1061,7 @@ def _iter_sse(response: Any) -> Any: try: name: str | None = None data_lines: list[str] = [] - for raw_line in response: + for raw_line in _iter_stream_lines(response): line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") if line == "": if name is not None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 0cce2f0c..af5d1812 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -23,6 +23,7 @@ import socket import threading import time +from http.client import IncompleteRead from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, ClassVar from urllib.parse import parse_qs, urlparse @@ -52,6 +53,7 @@ _retry_after_seconds, _SkillObjectSet, _store_object_from_put, + _StreamConnection, _tombstone_from_delete, ) @@ -1287,6 +1289,41 @@ def test_content_survives_a_reconnect(self, endpoint: Any) -> None: # --------------------------------------------------------------------------- +class _DyingResponse: + """ + A streaming body that transfers a payload and then fails mid-read. + + This is how a live stream actually ends: not with a clean end of body but + with a read timeout on a stream that went quiet, or a reset from the server + or a proxy in between. + """ + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def __iter__(self) -> Any: + for event in full_payload(("put-object", put_skill())): + yield f"event: {event['event']}\n".encode() + yield f"data: {json.dumps(event['data'])}\n".encode() + yield b"\n" + raise self._exc + + def close(self) -> None: + pass + + +class _DyingStreamRequester: + """Every connection transfers a payload, then dies with *exc* mid-read.""" + + def __init__(self, exc: BaseException) -> None: + self.connections = 0 + self._exc = exc + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + return _StreamConnection(_DyingResponse(self._exc)) + + class _ScriptedConnection: """Stands in for ``_StreamConnection``: an event iterator plus a close.""" @@ -1477,6 +1514,30 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize( + "exc", + [ + TimeoutError("timed out"), + ConnectionResetError(54, "Connection reset by peer"), + IncompleteRead(b"partial"), + ], + ids=["read timeout", "reset", "truncated body"], + ) + def test_a_stream_that_dies_mid_read_reconnects(self, exc: BaseException) -> None: + # A stream fails in its body far more often than at its connect, and + # ``read_timeout`` exists to bound one that has gone quiet. Treating + # such a failure as unexpected would stop delivery — including + # revocation — for the process lifetime the first time a socket died. + requester = _DyingStreamRequester(exc) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: requester.connections >= 5) + assert store.failed is None + finally: + store.close() + def test_a_stream_commit_resets_the_failure_count(self) -> None: payload = [ (e["event"], e["data"]) for e in full_payload(("put-object", put_skill())) From daf04a6e29720e9a20755f5768752e0bfe71187a Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Fri, 11 Sep 2026 16:14:39 -0400 Subject: [PATCH 25/43] fix(client): raise the skill content cap to 10 MiB The client-side cap was 64 KiB, close enough to the platform's own limit that any backend increase would force an SDK release. Raise it to 10 MiB so the guard stays a backstop against absurd input rather than a second enforcement of a bound this side does not own, and the real limit can grow without the SDKs moving. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 2 +- packages/client/agents.md | 2 +- .../client/src/launchdarkly_ai_server/skills_core.py | 8 +++++++- packages/client/tests/test_skills.py | 10 +++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index ae572b3b..8ad2985c 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -275,7 +275,7 @@ field of a different shape must rename it before upgrading. **Integrity is not optional.** Content is only returned after its sha256 (lowercase hex, over the verbatim UTF-8 bytes) matches the delivered `contentHash`, its key and version -revalidate, and its size is within 64 KiB. Anything that fails is withheld and treated as +revalidate, and its size is within 10 MiB. Anything that fails is withheld and treated as missing — no unverified content ever reaches your code. A retrieval that withheld anything logs a count at WARN, so a run that resolved nothing is not silent. diff --git a/packages/client/agents.md b/packages/client/agents.md index b5733567..2c28e64f 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -224,7 +224,7 @@ Store data is **untrusted input**; the transport is not part of the trust bounda themselves. - **Integrity is mandatory and doubled, through one implementation.** Every raw object is verified at the accessor boundary (key pattern and length, integer version >= 1, content - at most 64 KiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) + at most 10 MiB, sha256 lowercase hex over the verbatim bytes against `contentHash`) and the hash is re-verified immediately before a write, both through `skills_core.verified_bytes`, so the integrity signal's property set cannot depend on which layer caught the defect. A `Skill` is only ever constructed from content that diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index a3894335..3e7fabe3 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -67,11 +67,17 @@ ``launchdarkly_ai_server.skills_core``. """ -MAX_SKILL_CONTENT_BYTES = 64 * 1024 +MAX_SKILL_CONTENT_BYTES = 10 * 1024 * 1024 """ Hard cap on skill content. Legitimately delivered skills are well under this bound, so anything larger is withheld regardless of whether its hash checks out. +Set well above the platform's own limit on purpose. This is a backstop against +absurd input, not a second enforcement of the real bound: the platform refuses +oversized skills before they are ever delivered, and a client-side number sitting +just above that one would turn every backend increase into an SDK release. The +headroom lets the real limit grow without this constant moving. + Deliberately **not** exported from the package root, unlike the on-disk and on-the-wire constants beside it. Those are values this SDK defines and a caller may need to agree with; this one is a local enforcement bound on content the diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 0b0cadef..51e2217b 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -248,7 +248,7 @@ def test_content_cap_is_not_public_api(self) -> None: import launchdarkly_ai_server as package from launchdarkly_ai_server import skills_core - assert skills_core.MAX_SKILL_CONTENT_BYTES == 65536 + assert skills_core.MAX_SKILL_CONTENT_BYTES == 10485760 assert "MAX_SKILL_CONTENT_BYTES" not in package.__all__ assert not hasattr(package, "MAX_SKILL_CONTENT_BYTES") @@ -969,7 +969,7 @@ async def test_oversize_content_rejected( self, store: InMemorySkillStore, make_raw_skill: Any, recording_emitter: Any ) -> None: skills_module._set_emitter_for_testing(recording_emitter) - oversize = "x" * (64 * 1024 + 1) + oversize = "x" * (10 * 1024 * 1024 + 1) store.put(make_raw_skill(key="a", content=oversize)) assert await get_skill("a") is None assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 @@ -977,11 +977,11 @@ async def test_oversize_content_rejected( async def test_content_at_size_cap_is_accepted( self, store: InMemorySkillStore, make_raw_skill: Any ) -> None: - at_cap = "x" * (64 * 1024) + at_cap = "x" * (10 * 1024 * 1024) store.put(make_raw_skill(key="a", content=at_cap)) skill = await get_skill("a") assert skill is not None - assert len(skill.content) == 64 * 1024 + assert len(skill.content) == 10 * 1024 * 1024 async def test_key_at_length_bound_from_store_accepted( self, store: InMemorySkillStore, make_raw_skill: Any @@ -1132,7 +1132,7 @@ def _raw_without(field: str) -> dict[str, Any]: return raw -_OVERSIZE = "x" * (64 * 1024 + 1) +_OVERSIZE = "x" * (10 * 1024 * 1024 + 1) REASON_CODE_CASES = [ # Not a dict at all. Reachable through ``all_skills`` and not through From b7ee71a3477bad38d31123bed63e17a6203f3cbd Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:36:58 -0400 Subject: [PATCH 26/43] fix(client): keep FDv2 delivery alive on an idle stream, and shut down promptly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults in the delivery loop, all of which left the store reporting itself healthy while doing less than it claimed. **An up-to-date stream tripped the failure cap.** The consecutive-failure count reset only at a commit, and an environment whose skills are not changing answers every reconnect with `intentCode: "none"` and transfers nothing. A stream only ever ends by being dropped, so each recycle of a perfectly healthy idle connection counted as a failure — announced with a `goodbye` or not — and `max_consecutive_failures + 1` of them stopped delivery for the process lifetime, revocations included. The reset on commit covered only the case where content had changed, which is the case that was easy to test and not the case that runs in production. `_TransferOutcome` now reports `up_to_date`, and a complete answer that transfers nothing breaks the row of failures exactly as a commit does. An intent this module does not recognise is still not an answer. **`close` could not interrupt a poll.** The interrupt reached the streaming connection only, so polling parked in its request with nothing to reach and `close` returned when its join timed out — on a 300s-class request, long after the process meant to exit. `_Requester` now tracks the response of a poll in flight and offers `interrupt`, which `close` calls alongside the stream's own. A request still inside its connect has no response to reach; that one is bounded by `read_timeout`, and `start` no longer leaves the store inert when a join times out around it. An interrupt we asked for is no longer recorded as a delivery failure. **`close` left a waiter parked.** `wait_for_skills` waited on the first payload alone, so a shutdown racing a waiter added the waiter's whole timeout to it. Delivery ending is now its own event: a waiter is released by a payload, a give-up or a close, and reports whether a payload actually arrived rather than merely that it was let go. That also settles what `_give_up` had been quietly asserting — it set the first-payload flag to unblock waiters, which made `wait_for_skills` answer `True` for a store holding nothing. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 123 ++++++- packages/client/tests/test_skills_fdv2.py | 315 +++++++++++++++++- 2 files changed, 420 insertions(+), 18 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 18b81842..6a84092e 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -487,6 +487,14 @@ class _TransferOutcome: basis: str | None = None fatal: str | None = None disconnect: str | None = None + up_to_date: bool = False + """ + The server said what we hold is current and it has nothing to transfer. + + A complete answer that commits nothing, which is exactly what a 304 is to a + poll. The delivery loop counts it as a healthy connection; see + ``FDv2SkillStore._apply``. + """ class _ProtocolReader: @@ -576,6 +584,10 @@ def _server_intent(self, data: Any) -> _TransferOutcome: if intent != _INTENT_TRANSFER_NONE: logger.debug("Ignoring FDv2 server-intent with intentCode %r", intent) self._pending = None + # ``none`` is a complete answer that carries nothing. An intent this + # module does not recognise is not an answer at all, so only the + # former reports itself up to date. + return _TransferOutcome(up_to_date=intent == _INTENT_TRANSFER_NONE) return _TransferOutcome() def _target_for(self, data: Any) -> _SkillObjectSet | None: @@ -942,6 +954,25 @@ def __init__( self._read_timeout = read_timeout # Injectable so tests can drive a fake endpoint without a socket. self._opener = opener or urllib.request.build_opener() + self._lock = threading.Lock() + # The response of a poll in flight, so ``interrupt`` can reach its + # socket from another thread. Polling only: a stream's response is + # handed straight to the caller as a ``_StreamConnection``, which + # carries an interrupt of its own. + self._in_flight: Any = None + + def interrupt(self) -> None: + """ + Unblocks a poll parked in its body read, from another thread. + + Best effort, and safe to call when nothing is in flight. A request still + inside its connect has no response to reach yet and is bounded only by + ``read_timeout``; ``FDv2SkillStore.start`` covers what that leaves. + """ + with self._lock: + response = self._in_flight + if response is not None: + _interrupt_read(response) def _url(self, path: str, basis: str | None) -> str: """ @@ -972,11 +1003,17 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: request = self._request(POLL_PATH, basis, headers) try: with self._opener.open(request, timeout=self._read_timeout) as response: - status = getattr(response, "status", None) or response.getcode() - if status == 304: - return _PollResult(not_modified=True, events=[], etag=etag) - body = response.read() - new_etag = response.headers.get("ETag") or etag + with self._lock: + self._in_flight = response + try: + status = getattr(response, "status", None) or response.getcode() + if status == 304: + return _PollResult(not_modified=True, events=[], etag=etag) + body = response.read() + new_etag = response.headers.get("ETag") or etag + finally: + with self._lock: + self._in_flight = None except urllib.error.HTTPError as exc: if exc.code == 304: # urllib raises on 304 when no redirect handler swallows it. @@ -1225,6 +1262,18 @@ def __init__( self._stop = threading.Event() self._first_payload = threading.Event() + """A payload has committed. The fact ``wait_for_skills`` reports.""" + self._delivery_ended = threading.Event() + """ + Delivery has stopped, by ``close`` or by ``_give_up``. Kept apart from + ``_first_payload`` because it is not one: a waiter has to be let go + either way, but only a payload makes ``wait_for_skills`` true. + """ + self._released = threading.Event() + """ + Either of the two above, and what a waiter actually parks on: an + ``Event`` cannot wait on two, so the setters funnel through here. + """ self._thread: threading.Thread | None = None self._failed_reason: str | None = None # The open streaming connection, so ``close`` can interrupt its read. @@ -1244,7 +1293,13 @@ def start(self) -> FDv2SkillStore: Does not block: use ``wait_for_skills`` when boot ordering matters. """ with self._lock: + self._rearm_waiters() if self._thread is not None and self._thread.is_alive(): + # A ``close`` whose join timed out leaves the previous thread + # running with the stop flag still set. Clearing it lets that + # thread carry on delivering, rather than leaving a store that + # reports itself started and never delivers again. + self._stop.clear() return self self._stop.clear() self._thread = threading.Thread( @@ -1253,6 +1308,16 @@ def start(self) -> FDv2SkillStore: self._thread.start() return self + def _rearm_waiters(self) -> None: + """ + Re-arms ``wait_for_skills`` for a store being started again after a + ``close``. A payload already held stays an answer; an ended delivery + does not, or the next waiter would be released before it began. + """ + self._delivery_ended.clear() + if not self._first_payload.is_set(): + self._released.clear() + def close(self, timeout: float = 5.0) -> None: """ Stops delivery. Idempotent, and safe to call from any thread. @@ -1262,12 +1327,17 @@ def close(self, timeout: float = 5.0) -> None: package-level ``launchdarkly_ai_server.shutdown()`` coroutine. """ self._stop.set() + # A waiter parked in ``wait_for_skills`` is owed an answer now rather + # than at the end of its timeout; delivery is over either way. + self._end_delivery() # The delivery thread is normally blocked in a socket read that no flag - # can reach; without this the join waits out its full timeout. + # can reach; without this the join waits out its full timeout. Streaming + # parks in the connection, polling in the request, so interrupt both. with self._lock: connection = self._connection if connection is not None: connection.close() + self._requester.interrupt() thread = self._thread if ( thread is not None @@ -1288,8 +1358,24 @@ def wait_for_skills(self, timeout: float = 10.0) -> bool: ``True`` means a payload arrived — not that any skill in it verified, and not that the environment has any skills. ``diagnostics`` answers the rest. + + Returns early, ``False``, when delivery ends before any payload does: + a ``close`` from another thread, or a failure delivery cannot retry. + Waiting out the full timeout for an answer that has already arrived + would delay every shutdown that raced a waiter. """ - return self._first_payload.wait(timeout=timeout) + self._released.wait(timeout=timeout) + return self._first_payload.is_set() + + def _publish_first_payload(self) -> None: + """Records the first committed payload and lets any waiter go.""" + self._first_payload.set() + self._released.set() + + def _end_delivery(self) -> None: + """Records that delivery has stopped and lets any waiter go.""" + self._delivery_ended.set() + self._released.set() @property def failed(self) -> str | None: @@ -1383,6 +1469,11 @@ def _run(self) -> None: self._give_up(str(exc)) return except _RecoverableTransportError as exc: + if self._stop.is_set(): + # ``close`` interrupted the request on purpose. Counting it + # would spend a retry from the bounded budget and leave a + # misleading ``last_error`` on a healthy store. + return with self._lock: self._failures += 1 failures = self._failures @@ -1432,8 +1523,8 @@ def _give_up(self, reason: str) -> None: "the process restarts with a working connection.", reason, ) - # Unblock anyone waiting on a first payload that is never coming. - self._first_payload.set() + # Let go of anyone waiting on a first payload that is never coming. + self._end_delivery() def _apply(self, name: str, data: Any) -> None: """ @@ -1444,10 +1535,16 @@ def _apply(self, name: str, data: Any) -> None: outcome = self._reader.handle(name, data) if outcome.committed and outcome.basis is not None: self._basis = outcome.basis - if outcome.committed: - # A commit breaks the row of consecutive failures. + if outcome.committed or outcome.up_to_date: + # Both break the row of consecutive failures: a commit is a payload + # delivered, and ``up_to_date`` is the server confirming we already + # hold it. Counting only the commit would give up on a healthy + # stream serving an environment whose skills are not changing: + # nothing to transfer means no commit, while every recycled + # connection still ends in a drop. self._record_success() - self._first_payload.set() + if outcome.committed: + self._publish_first_payload() if outcome.changes: self._notify(outcome.changes) if outcome.fatal: @@ -1465,7 +1562,7 @@ def _poll_once(self) -> None: logger.debug("Skill payload unchanged (HTTP 304)") # A 304 counts as a first payload, so a boot that reconnects with a # cached basis is not blocked on a transfer the server will not send. - self._first_payload.set() + self._publish_first_payload() return for name, data in result.events: self._apply(name, data) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index af5d1812..d063e451 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -538,6 +538,17 @@ def test_a_full_transfer_commits_at_payload_transferred(self) -> None: assert outcomes[-1].committed is True assert outcomes[-1].basis == "basis-1" + def test_an_up_to_date_intent_is_reported_as_such(self) -> None: + """``intentCode: "none"`` is the stream's 304: current, nothing to send.""" + reader = _ProtocolReader(_SkillObjectSet()) + outcome = reader.handle("server-intent", server_intent("none")) + assert outcome.up_to_date is True + assert outcome.committed is False + assert outcome.disconnect is None + # A transfer intent is a promise of content, not an up-to-date answer. + transfer = reader.handle("server-intent", server_intent("xfer-full")) + assert transfer.up_to_date is False + def test_nothing_is_visible_before_payload_transferred(self) -> None: """A payload version is the unit of consistency; half of one is not a state.""" held = _SkillObjectSet() @@ -1312,7 +1323,17 @@ def close(self) -> None: pass -class _DyingStreamRequester: +class _FakeRequester: + """ + Base for the requester fakes: supplies the ``interrupt`` the store calls on + ``close``, so each fake only scripts the part it is about. + """ + + def interrupt(self) -> None: + """No real socket to reach; these fakes end their own connections.""" + + +class _DyingStreamRequester(_FakeRequester): """Every connection transfers a payload, then dies with *exc* mid-read.""" def __init__(self, exc: BaseException) -> None: @@ -1335,7 +1356,7 @@ def close(self) -> None: self.closed = True -class _ScriptedRequester: +class _ScriptedRequester(_FakeRequester): """Raises a scripted sequence, so backoff is asserted without real sockets.""" def __init__(self, *outcomes: Any) -> None: @@ -1361,7 +1382,7 @@ def stream(self, basis: str | None) -> Any: return _ScriptedConnection(outcome) -class _RecyclingRequester: +class _RecyclingRequester(_FakeRequester): """ A healthy server that recycles connections: every ``stream`` call succeeds, transfers a full payload, and then ends the connection, as LaunchDarkly and @@ -1383,6 +1404,53 @@ def stream(self, basis: str | None) -> Any: ) +class _UpToDateRecyclingRequester(_FakeRequester): + """ + A healthy server with nothing new to say: every connection answers + ``intentCode: "none"`` — the stream's equivalent of a 304 — transfers + nothing, and is then recycled. This is the steady state of an environment + whose skills are not changing, which is most environments most of the time. + """ + + def __init__(self, farewell: bool = False) -> None: + self.connections = 0 + self._farewell = farewell + + def stream(self, basis: str | None) -> Any: + self.connections += 1 + script: list[tuple[str, Any]] = [ + ("server-intent", server_intent("none")), + ("heart-beat", {}), + ] + if self._farewell: + # A recycle is often announced rather than abrupt. + script.append(("goodbye", {"reason": "connection recycled"})) + return _ScriptedConnection(script) + + +class _SlowPollRequester(_FakeRequester): + """ + A poll whose request does not return until the test releases it, standing in + for one blocked where no interrupt can reach: inside its connect. + """ + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def poll(self, basis: str | None, etag: str | None) -> Any: + self.entered.set() + self.release.wait(timeout=10) + raise _RecoverableTransportError("released") + + +class _SilentStreamRequester(_FakeRequester): + """A stream that connects and then delivers nothing until it is closed.""" + + def stream(self, basis: str | None) -> Any: + return _BlockingConnection() + + class _BlockingConnection: """A stream that never produces an event until it is closed.""" @@ -1398,7 +1466,7 @@ def close(self) -> None: self._closed.set() -class _SlowConnectRequester: +class _SlowConnectRequester(_FakeRequester): """ A ``stream`` whose connect does not return until the test releases it, standing in for a slow TLS handshake, followed by a read that never yields. @@ -1447,7 +1515,11 @@ def test_a_fatal_failure_releases_wait_for_skills_rather_than_hanging( ) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: - assert store.wait_for_skills(timeout=5) is True + started = time.monotonic() + # Released promptly, and ``False``: no payload arrived, and saying + # otherwise would send a caller on to read a store holding nothing. + assert store.wait_for_skills(timeout=5) is False + assert time.monotonic() - started < 2.0 assert store.failed is not None def test_a_fatal_failure_keeps_last_known_good_servable( @@ -1514,6 +1586,28 @@ def test_recycled_stream_connections_are_not_failures(self) -> None: finally: store.close() + @pytest.mark.parametrize("farewell", [False, True], ids=["dropped", "goodbye"]) + def test_an_up_to_date_recycled_stream_is_not_a_failure( + self, farewell: bool + ) -> None: + # Resetting at a commit covers only a connection that carried new + # content. An environment whose skills are not changing answers every + # reconnect with ``intentCode: "none"`` and transfers nothing, so a loop + # that counted those drops would give up on a *healthy* idle stream + # after max_consecutive_failures + 1 recycles — and revocation, the one + # thing streaming exists to deliver promptly, would never arrive again. + requester = _UpToDateRecyclingRequester(farewell=farewell) + store = stream_store(max_consecutive_failures=3, _requester=requester) + try: + store.start() + assert wait_until(lambda: requester.connections >= 8) + assert store.failed is None + # As with a payload-carrying recycle, the count may read 1 + # mid-reconnect. What it must never do is climb. + assert store.diagnostics.connection_failures <= 1 + finally: + store.close() + @pytest.mark.parametrize( "exc", [ @@ -2194,6 +2288,64 @@ def close(self) -> None: self._listener.close() +class _StalledBody: + """ + A listening socket that answers with headers and then stalls the body. + + Distinct from ``_BlackHole``: here the request succeeds far enough to hand + urllib a response, and the caller then parks in ``read``. That is the state + ``close`` has to interrupt — and, unlike a request still inside its connect, + the state an interrupt can actually reach. + """ + + def __init__(self) -> None: + self._listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(8) + self._accepted: list[socket.socket] = [] + self.serving = threading.Event() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve_forever, daemon=True) + self._thread.start() + host, port = self._listener.getsockname() + self.base_uri = f"http://{host}:{port}" + + def _serve_forever(self) -> None: + self._listener.settimeout(0.05) + while not self._stop.is_set(): + try: + conn, _ = self._listener.accept() + except OSError: + continue + self._accepted.append(conn) + try: + conn.recv(4096) + # A length far longer than the body that follows, so the read + # blocks rather than seeing the end of the message. + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 4096\r\n\r\n" + ) + except OSError: + continue + self.serving.set() + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=2) + for conn in self._accepted: + conn.close() + self._listener.close() + + +@pytest.fixture +def stalled_body() -> Any: + server = _StalledBody() + yield server + server.close() + + @pytest.fixture def black_hole() -> Any: server = _BlackHole() @@ -2274,3 +2426,156 @@ def test_there_is_no_separate_connect_timeout(self) -> None: # constructor does not offer a parameter that would only pretend to. with pytest.raises(TypeError): FDv2SkillStore(SDK_KEY, connect_timeout=2.0) # type: ignore[call-arg] + + +class TestWaitingForSkills: + """ + ``wait_for_skills`` answers with what happened, and never outlives it. + + Its budget is a boot-ordering allowance, not a delay to spend: a store that + already knows no payload is coming owes the caller that answer immediately. + """ + + def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: + # A shutdown racing a waiter is the ordinary case, not an exotic one: + # ``close`` on the main thread while a worker is still waiting for its + # first payload. Parking that worker for the rest of its timeout adds + # the whole budget to a process that has already decided to stop. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + answers: list[bool] = [] + waiter = threading.Thread( + target=lambda: answers.append(store.wait_for_skills(timeout=10)), + daemon=True, + ) + waiter.start() + time.sleep(0.2) + started = time.monotonic() + store.close() + waiter.join(timeout=5) + assert not waiter.is_alive() + assert time.monotonic() - started < 2.0 + assert answers == [False] + + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: + # ``close`` does not drop content, so it must not turn the answer about + # that content into a lie either. + requester = _RecyclingRequester() + store = stream_store(_requester=requester) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + finally: + store.close() + assert store.wait_for_skills(timeout=5) is True + + def test_a_restarted_store_waits_again(self) -> None: + # The released flag is sticky by design, so a store closed before any + # payload and then started again has to re-arm: otherwise the next + # waiter is let go before delivery has had a chance to begin. + store = stream_store(_requester=_SilentStreamRequester()) + store.start() + store.close() + assert store.wait_for_skills(timeout=0.1) is False + store.start() + try: + started = time.monotonic() + assert store.wait_for_skills(timeout=0.5) is False + # Waited, rather than being released by the previous close. + assert time.monotonic() - started >= 0.4 + finally: + store.close() + + +class TestPollShutdown: + """ + ``close`` has to interrupt a poll in flight, as it already does a stream. + + Without it the delivery thread stays parked in its request and ``close`` + returns only when the join times out — on a 300s-class request, long after + the process meant to exit. The bound is loose on purpose: the point is + promptly rather than a particular number of milliseconds. + """ + + def test_interrupt_unblocks_a_poll_stalled_in_its_body( + self, stalled_body: Any + ) -> None: + requester = _Requester(SDK_KEY, stalled_body.base_uri, read_timeout=30.0) + raised: list[BaseException] = [] + + def poll_until_interrupted() -> None: + try: + requester.poll(None, None) + except BaseException as exc: + raised.append(exc) + + thread = threading.Thread(target=poll_until_interrupted, daemon=True) + thread.start() + assert stalled_body.serving.wait(timeout=5) + # The response is in hand; give the read a moment to park in it. + time.sleep(0.2) + started = time.monotonic() + requester.interrupt() + thread.join(timeout=5) + assert not thread.is_alive() + assert time.monotonic() - started < 2.0 + assert raised and isinstance(raised[0], _RecoverableTransportError) + + def test_close_during_a_stalled_poll_returns_promptly( + self, stalled_body: Any + ) -> None: + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + started = time.monotonic() + store.close(timeout=5.0) + assert time.monotonic() - started < 2.0 + assert store._thread is not None and not store._thread.is_alive() + + def test_a_poll_we_interrupted_is_not_a_delivery_failure( + self, stalled_body: Any + ) -> None: + # Our own shutdown is not an outage: counting it would spend a retry + # from the bounded budget and leave a misleading ``last_error`` behind + # on a store whose content is still perfectly good. + store = FDv2SkillStore( + SDK_KEY, + base_uri=stalled_body.base_uri, + mode="poll", + poll_interval=0.05, + read_timeout=30.0, + ) + store.start() + assert stalled_body.serving.wait(timeout=5) + time.sleep(0.2) + store.close(timeout=5.0) + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + assert store.failed is None + + def test_a_close_that_timed_out_leaves_the_store_restartable(self) -> None: + # A request blocked inside its connect is beyond any interrupt, so + # ``close`` can still return with the thread alive. ``start`` must not + # then find that thread and return with the stop flag set: the store + # would report itself started and never deliver again. + requester = _SlowPollRequester() + store = FDv2SkillStore( + SDK_KEY, mode="poll", poll_interval=0.01, _requester=requester + ) + try: + store.start() + assert requester.entered.wait(timeout=5) + store.close(timeout=0.2) + assert store._thread is not None and store._thread.is_alive() + store.start() + assert store._stop.is_set() is False + finally: + requester.release.set() + store.close(timeout=2) From 88c225ebc296613713d32fcfa3f7fb474ffe6495 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Mon, 14 Sep 2026 14:47:44 -0400 Subject: [PATCH 27/43] fix(client): log a recycled FDv2 stream at debug rather than warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream only ever ends by being dropped, and LaunchDarkly — and any proxy in between — recycles a long-lived one. Every reconnect therefore logged "Skill delivery failed" at WARNING, for as long as the process ran. Until the previous commit that noise was bounded, because an idle stream gave up after eleven recycles and went quiet; now that delivery correctly survives them, it would run forever and describe a healthy store as failing. A connection that got a complete answer before it ended — a committed payload, or an up-to-date intent — delivered everything it was asked for, so its reconnect is now DEBUG and says so. A connection that ended without answering is the case the warning exists for and still gets it: a connect that never landed, or a transfer that died part-way through. Filling a customer's logs with a fault they do not have is not merely untidy; it teaches them that the level which means something can be ignored. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 27 ++++++++++++++-- packages/client/tests/test_skills_fdv2.py | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6a84092e..71411c8a 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1283,6 +1283,10 @@ def __init__( # by being dropped, so resetting on return would count every healthy, # server-recycled connection as a failure. self._failures = 0 + # Whether the current attempt got a complete answer before it ended. + # A stream only ever ends by being dropped, so this is what separates + # a recycled healthy connection from one that failed. + self._attempt_answered = False # -- lifecycle --------------------------------------------------------- @@ -1456,6 +1460,8 @@ def _notify(self, changes: list[dict[str, Any]]) -> None: def _run(self) -> None: while not self._stop.is_set(): + with self._lock: + self._attempt_answered = False try: if self._mode == "stream": self._stream_once() @@ -1477,6 +1483,7 @@ def _run(self) -> None: with self._lock: self._failures += 1 failures = self._failures + answered = self._attempt_answered self._reader.diagnostics.connection_failures = failures self._reader.diagnostics.last_error = str(exc) if failures > self._max_consecutive_failures: @@ -1494,9 +1501,22 @@ def _run(self) -> None: # The header may come from a proxy rather than LaunchDarkly, and # a value in the hours would park revocation for that long. delay = min(delay, self._max_backoff) - logger.warning( - "Skill delivery failed (%s); retrying in %.1fs", exc, delay - ) + if answered: + # LaunchDarkly, and any proxy in between, recycles a + # long-lived stream. A connection that answered before it + # ended delivered everything it was asked for, so the + # reconnect is routine rather than a fault worth warning + # about for as long as the process runs. + logger.debug( + "The FDv2 stream ended after a complete answer (%s); " + "reconnecting in %.1fs", + exc, + delay, + ) + else: + logger.warning( + "Skill delivery failed (%s); retrying in %.1fs", exc, delay + ) if self._stop.wait(delay): return continue @@ -1511,6 +1531,7 @@ def _run(self) -> None: def _record_success(self) -> None: with self._lock: self._failures = 0 + self._attempt_answered = True self._reader.diagnostics.connection_failures = 0 def _give_up(self, reason: str) -> None: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index d063e451..b0c6aa86 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1608,6 +1608,38 @@ def test_an_up_to_date_recycled_stream_is_not_a_failure( finally: store.close() + def test_a_recycled_connection_reconnects_quietly(self, caplog: Any) -> None: + # A healthy idle stream reconnects for as long as the process runs, so + # warning on each one would fill a customer's logs with a fault they do + # not have and teach them to ignore the level that means something. + requester = _UpToDateRecyclingRequester() + store = stream_store(_requester=requester) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: requester.connections >= 5) + finally: + store.close() + assert store.failed is None + assert not [r for r in caplog.records if r.levelname == "WARNING"] + assert [r for r in caplog.records if "reconnecting in" in r.getMessage()] + + def test_a_connection_that_never_answered_still_warns(self, caplog: Any) -> None: + # The quiet path is earned by answering. A connection that failed before + # it told us anything is the case the warning exists for. + store = stream_store( + max_consecutive_failures=10, _requester=_ScriptedRequester() + ) + with caplog.at_level("DEBUG", logger="launchdarkly_ai_server.skills_fdv2"): + try: + store.start() + assert wait_until(lambda: store.diagnostics.connection_failures >= 3) + finally: + store.close() + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert warnings + assert all("Skill delivery failed" in r.getMessage() for r in warnings) + @pytest.mark.parametrize( "exc", [ From d3a8bc1c01170f6523821ac31ad73675ee47b12e Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 11:03:42 -0400 Subject: [PATCH 28/43] fix(client): read the skills manifest through the pinned root descriptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconcile pinned the root for every destructive step but still read the manifest by path, and the manifest is the input that authorizes those steps. A root swapped after the pin therefore supplied the entries for the whole run, and the consequences landed in the real root through the descriptor. The damaging case is delayed rather than immediate. `_unsafe_path_reason` refuses a write or a prune whose path resolves out of the validated root, so the swapped run itself reports refusals and deletes nothing. What it cannot refuse is the record: `_rewrite_manifest` commits the entries that were read back over the real manifest, so the run ends with the manifest claiming a file the SDK never wrote — and the *next* reconcile, an ordinary one with no attacker present and every path check passing, removes the customer's file. This is also the one respect in which pinning the root made things worse. Before the pin, the manifest write was the only operation that took its own `O_NOFOLLOW` descriptor, so a swapped root made that write fail and left the real manifest intact. `_load_manifest` now reads through the held descriptor, reusing `_read_regular_file` so the manifest gets the same open the skill files get: `O_NOFOLLOW` refuses a symlink wearing its name, `O_NONBLOCK` plus the `S_ISREG` check on the descriptor refuse a FIFO instead of blocking the event loop on it, and the single open replaces an `exists()`-then-read pair on the same path. `max_bytes=None` reads to EOF, since the manifest is parsed rather than compared against a known length. Reads under the root on the write path are unchanged and still noted as out of scope: `_write_one`'s comparison read names `/SKILL.md`, so covering it needs a descriptor for the skill directory rather than for the root. Two tests, both verified to fail with the source change reverted: - a root swapped between the pin and the manifest read cannot supply the entries — the real manifest still records what it owned; - the same swap cannot poison the ownership record — asserted across both phases, because the deletion happens on the following clean run. Reported by Cursor Bugbot on #70. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fs.py | 81 +++++++--- packages/client/tests/test_skills_fs.py | 142 ++++++++++++++++++ 2 files changed, 204 insertions(+), 19 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 2666bee1..2c098995 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -170,8 +170,11 @@ async def write_skills( The root is opened once, up front, and that descriptor is held until the call returns: every directory created, file written and file removed under it is resolved relative to the descriptor rather than from the root's path, - so a root replaced after validation is refused rather than followed. This - guarantee is POSIX-only, for the reasons ``safe_fs`` states at length. + so a root replaced after validation is refused rather than followed. The + manifest is read through it too, so the record that authorizes those + removals comes from inside the pinned root rather than from wherever the + root's path led by the time it was read. This guarantee is POSIX-only, for + the reasons ``safe_fs`` states at length. **This call performs synchronous filesystem I/O and does not yield.** It is ``async`` for signature parity with the other accessors and with the @@ -227,7 +230,7 @@ async def write_skills( ) try: - manifest, manifest_error = _load_manifest(root_path) + manifest, manifest_error = _load_manifest(root_path, root_fd) entries: dict[str, Any] = manifest.get("entries", {}) actions: list[ReconcileAction] = [] @@ -615,7 +618,9 @@ def _resolve_root(root: str | os.PathLike[str]) -> Path: return Path(os.path.realpath(path)) -def _load_manifest(root: Path) -> tuple[dict[str, Any], str | None]: +def _load_manifest( + root: Path, root_fd: int | None +) -> tuple[dict[str, Any], str | None]: """ Loads the manifest. Returns ``(manifest, error)``. @@ -627,16 +632,41 @@ def _load_manifest(root: Path) -> tuple[dict[str, Any], str | None]: mean guessing at which of the customer's files are ours. An absent manifest is not corrupt — that is simply a fresh root. + + Read relative to the root descriptor the caller holds, for the same reason + every destructive step is: this file is what decides which of the + customer's files the SDK may overwrite and delete. Read by path it + re-resolved the root on the way in, so a root swapped after the pin handed + the run somebody else's entries — and every consequence then landed in the + *real* root through the descriptor. An empty manifest from the wrong + directory made every managed file look unowned; a populated one aimed the + prune; and either way ``_rewrite_manifest`` committed the result back over + the real manifest, destroying the ownership record that protects the + customer's files on the next run. Reading it through the descriptor is what + makes the pin cover the decision as well as the actions. + + The single descriptor-relative open also replaces an ``exists()``-then-read + pair on the same path, so absence is now ``ENOENT`` on the read itself. It + is the same open the skill files get, which additionally means a symlink or + a FIFO wearing the manifest's name is refused as corruption rather than + followed or waited on. """ - path = root / MANIFEST_FILENAME fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} - if not path.exists(): + try: + raw = _read_regular_file( + MANIFEST_FILENAME if root_fd is not None else root / MANIFEST_FILENAME, + max_bytes=None, + dir_fd=root_fd, + ) + except FileNotFoundError: return fresh, None + except OSError as exc: + return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" try: - text = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError) as exc: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: # UnicodeDecodeError is a ValueError, not an OSError: non-UTF-8 bytes in # the manifest are corruption, and must fail closed like any other. return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" @@ -867,7 +897,9 @@ def failed(message: str) -> ReconcileAction: ) -def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: +def _read_regular_file( + target: Path | str, *, max_bytes: int | None, dir_fd: int | None = None +) -> bytes: """ Reads *target*, refusing anything that is not a regular file. @@ -882,11 +914,21 @@ def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: without it translates CRLF on read, which would fail the hash comparison against content that is actually current. - Reads at most ``max_bytes + 1`` bytes. The only consumer compares a hash, and - anything longer than the resolved content cannot match it, so the one extra - byte is enough to prove inequality — which is what keeps a foreign file of - arbitrary size from being pulled into memory now that adoption reads files - the manifest does not list. + With *dir_fd*, *target* is a bare component resolved against that + descriptor rather than a path the kernel walks from the root down. The + manifest read passes the root descriptor, because the manifest is what + decides which of the customer's files the SDK may touch. ``_write_one``'s + comparison read still resolves a path: what it names is + ``/SKILL.md``, so covering it the same way needs a descriptor for the + skill directory rather than for the root. + + ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes: the comparison + consumer only ever hashes the result, and anything longer than the resolved + content cannot match it, so the one extra byte is enough to prove + inequality — which is what keeps a foreign file of arbitrary size from being + pulled into memory now that adoption reads files the manifest does not + list. ``None`` reads to EOF, for the manifest, whose length no caller can + predict and which is parsed rather than compared. """ flags = ( os.O_RDONLY @@ -894,18 +936,19 @@ def _read_regular_file(target: Path, *, max_bytes: int) -> bytes: | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0) ) - fd = os.open(target, flags) + fd = os.open(target, flags, dir_fd=dir_fd) try: if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError("the target file is not a regular file") chunks: list[bytes] = [] - remaining = max_bytes + 1 - while remaining > 0: - chunk = os.read(fd, min(remaining, 65536)) + remaining = None if max_bytes is None else max_bytes + 1 + while remaining is None or remaining > 0: + chunk = os.read(fd, 65536 if remaining is None else min(remaining, 65536)) if not chunk: break chunks.append(chunk) - remaining -= len(chunk) + if remaining is not None: + remaining -= len(chunk) return b"".join(chunks) finally: os.close(fd) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index bd891b1f..2ece6a57 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -1314,6 +1314,148 @@ def resolve_then_swap(argument: Any) -> Path: assert not (outside / "a" / MANIFEST_NAME).exists() assert list(moved_to.iterdir()) == [] + @staticmethod + def _swap_after_the_pin( + monkeypatch: pytest.MonkeyPatch, root: Path, outside: Path, moved_to: Path + ) -> None: + """Swaps the root in the window between the pin and the manifest read. + + The three races above intercept a destructive call. This one intercepts + the pin itself and swaps the root the instant it returns, which is the + earliest point the held descriptor is already in hand — so everything + the reconcile *decides*, not just everything it does, happens with a + hostile root on the path. The manifest is the first thing read in that + window, and it is the only input that says which of the customer's + files the SDK may overwrite and delete. + """ + real_pin = skills_fs_module.open_directory_nofollow + + def pin_then_swap(path: Any) -> int | None: + fd = real_pin(path) + os.rename(root, moved_to) + os.symlink(outside, root, target_is_directory=True) + return fd + + monkeypatch.setattr(skills_fs_module, "open_directory_nofollow", pin_then_swap) + + @_needs_dir_fd + async def test_a_root_swapped_before_the_manifest_read_cannot_supply_the_entries( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The manifest is read through the descriptor, not through the path. + + Read by path, the manifest was the one input to the reconcile that the + pin did not cover: the swap redirected the read into the attacker's + directory, the run adopted whatever entries it found there, and + ``_rewrite_manifest`` then committed them back over the *real* manifest + through the held descriptor. Nothing escaped the root — but the + ownership record inside it was destroyed, and that record is the only + thing standing between the next reconcile and the customer's own files. + Note that this is strictly worse than the behavior it replaced: before + the root was pinned at all, the manifest write was the one operation + that took its own ``O_NOFOLLOW`` descriptor, so a swapped root made the + write *fail* and left the real manifest intact. + + The attacker's manifest here is valid and empty, which is the cheapest + version to plant and the one that does the most damage: every managed + file looks unowned, so the entry recording it is simply dropped on the + rewrite. ``prune`` is off so that what this asserts is the entries the + run read, uncoupled from what a prune driven by them would then remove + — which is the next test. + + The write of the requested skill is refused either way, by the path + checks: they resolve ``/other`` into the attacker's tree and see + it land outside the managed root. That refusal is what makes this test + narrow rather than weaker — with no write and no prune, the manifest + rewrite is the only thing left in the run, so the surviving entry can + only have come from reading the real manifest. It is also the two + layers doing the jobs they are each documented to do: the path checks + turn a hostile layout into a reported refusal, and the descriptor is + what makes the refusal unnecessary for correctness. + """ + root = tmp_path / "skills" + root.mkdir() + _place_managed(root, "keep", SKILL_BODY) + outside = tmp_path / "outside" + outside.mkdir() + _write_manifest(outside, {"manifestVersion": 1, "entries": {}}) + moved_to = tmp_path / "skills.real" + + self._swap_after_the_pin(monkeypatch, root, outside, moved_to) + + report = await write_skills([_skill("other")], root, prune=False) + + # The real root is where the rename left it, and its manifest still + # records the skill it owned before the run. + assert "keep/SKILL.md" in _read_manifest(moved_to)["entries"] + # The attacker's manifest is neither the one that was read nor the one + # that was written. + assert _read_manifest(outside)["entries"] == {} + # The refusal named above, asserted so that a future change which + # starts writing through the hostile path does not pass this quietly. + assert [a.key for a in report.errors] == ["other"], _error_messages(report) + + @_needs_dir_fd + async def test_a_root_swapped_before_the_manifest_read_cannot_poison_ownership( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The delayed half of the same window, and the damaging one. + + A populated manifest in the attacker's directory does not get to delete + anything *during* the swapped run: the path checks resolve + ``/victim`` into the attacker's tree, see it land outside the + managed root, and refuse the prune. What they cannot refuse is the + record. ``_rewrite_manifest`` commits the entries that were read back + into the real root through the held descriptor, so the swapped run ends + with the real manifest claiming a file the SDK never wrote. + + The deletion then happens on the *next* reconcile — an ordinary one, + with no attacker present and every path check passing, because by then + the entry is in the legitimate manifest, the key is well formed, and the + path really is inside the real root. That is what makes reading the + manifest by path worth fixing rather than noting: the blast radius is + not the swapped run, it is every run after it, and the report for the + run that did the damage shows only a refusal. + + Both phases run here for that reason. Asserting only that the entry is + absent after phase one would leave the consequence implicit, and the + consequence is a customer's own file. + """ + root = tmp_path / "skills" + root.mkdir() + victim = root / "victim" / "SKILL.md" + victim.parent.mkdir() + victim.write_text("the customer's own file\n", encoding="utf-8") + outside = tmp_path / "outside" + (outside / "victim").mkdir(parents=True) + # Present so the path-based existence probe on the prune path is + # satisfied; without it the run would skip the unlink for a reason that + # has nothing to do with the defense under test. + (outside / "victim" / "SKILL.md").write_text("bait\n", encoding="utf-8") + _write_manifest( + outside, + { + "manifestVersion": 1, + "entries": {"victim/SKILL.md": _entry("victim", 1, SKILL_BODY)}, + }, + ) + moved_to = tmp_path / "skills.real" + + self._swap_after_the_pin(monkeypatch, root, outside, moved_to) + await write_skills([_skill("a")], root) + monkeypatch.undo() + + # The attacker withdraws, restoring the root exactly as it was found. + os.unlink(root) + os.rename(moved_to, root) + assert "victim/SKILL.md" not in _read_manifest(root)["entries"] + + # An ordinary reconcile, which is where the planted entry would cash in. + report = await write_skills([], root) + + assert [a for a in report.actions if a.action == "removed"] == [] + assert victim.read_text(encoding="utf-8") == "the customer's own file\n" + @_needs_dir_fd async def test_every_destructive_call_runs_relative_to_a_descriptor( self, root: Path, monkeypatch: pytest.MonkeyPatch From b7ad16f80985142e571bf142606b1be592a04e64 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 12:51:45 -0400 Subject: [PATCH 29/43] fix(client): give the key-mismatch resolution its outcome reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolve_from_store` withholds an answer served under a key other than the one asked for — identity is read off the object itself, so a substituted skill would otherwise be handed back under the caller's key while carrying its own. That construction site was left without a `reason` when #58 made the field required, so the path raised `TypeError` instead of returning a `Resolution`: a store substituting one skill for another crashed the caller rather than being withheld from it, which is the opposite of what the check is for. Three tests were failing on it (`test_a_store_answering_under_a_different_key_is_withheld` and the two `..._another_key_...` cases in `test_skills_fs.py`), along with `mypy`, which caught the missing argument exactly as the no-default design intended. The token is `integrity_failure`. Content was delivered and its identity did not verify, and that is the one outcome a caller is expected to fail closed on; reporting `absent` would file a substituting store in the bucket the same caller is invited to tolerate. It is not `wrong_version` either — that names a version mismatch specifically, and there is deliberately no `wrong_key` to parallel it. The choice is not recoverable from the error message, so a test now pins it rather than leaving it to the next reader. Two unrelated failures remain on this branch, both from the content cap moving to 10 MB while the tests still build their oversize fixture at 64 KiB + 1: `test_oversize_skill_aborts_the_write` and `test_integrity_signal_property_keys_match_across_layers`. Left alone — that change is in flight elsewhere and its owner should decide whether the fixture or the cap is what moved. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_core.py | 10 ++++- packages/client/tests/test_skills.py | 37 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index db15e943..3215f394 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -779,11 +779,19 @@ def resolve_from_store( error=f"skill '{key}' failed integrity verification and was withheld", ) if skill.key != key: + # ``integrity_failure`` rather than ``absent``: content was delivered + # and its identity did not verify, which is the one token a caller is + # expected to fail closed on. Reporting ``absent`` would file a store + # that substitutes one skill for another in the bucket the same caller + # is invited to tolerate. It is not ``wrong_version`` either — that + # token names a version mismatch specifically, and there is deliberately + # no ``wrong_key`` to parallel it. return Resolution( + reason="integrity_failure", error=( f"skill '{key}' is not available: the store answered under " f"key '{skill.key}'" - ) + ), ) if wanted_version is not None and skill.version != wanted_version: return Resolution( diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index c72930fa..db709ab6 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -935,6 +935,43 @@ async def test_integrity_failure_when_content_does_not_verify( # letter that appears in half the words in the message. assert "'a'" in outcome.detail + async def test_integrity_failure_when_the_store_answers_under_another_key( + self, make_raw_skill: Any + ) -> None: + """Identity is part of verification, so a substitution fails closed. + + Grouped with the tampered-content case above rather than with + ``absent``, and the grouping is the assertion: the store did answer, it + answered with a skill, and what disqualified the answer was its + identity. A caller that fails closed on suspected tampering has to see + that. Filing it as ``absent`` would put a store substituting one skill + for another in the bucket that same caller is invited to tolerate. + + Pinned in a test because the token is not recoverable from the message + and the construction site went without one long enough to raise + ``TypeError`` on this path. + """ + + class _AliasingStore: + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> Any: + return make_raw_skill(key="other-key") + + def all_objects(self, kind: str) -> dict[str, Any]: + return {} + + skills_module._set_store(_AliasingStore()) + + outcome = await get_skill_result("asked-for") + + assert outcome.reason == "integrity_failure" + assert outcome.skill is None + assert outcome.detail + # Both keys, since the detail is what makes a substitution diagnosable. + assert "'asked-for'" in outcome.detail + assert "'other-key'" in outcome.detail + async def test_wrong_version_when_the_store_answers_with_another( self, make_raw_skill: Any ) -> None: From 0902f29eff50681bcbc0dda5cdf841e0c67cb596 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 13:18:33 -0400 Subject: [PATCH 30/43] test(client): derive the oversize fixture from the content cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both consumers built their oversize content as `"x" * (64 * 1024 + 1)`, a literal pinned to the old 64 KiB bound. When the cap moved to 10 MiB the literal stayed, which put the fixture *under* the limit and inverted both tests: `test_oversize_skill_aborts_the_write` asserted that oversize content is refused while handing verification something it should accept, and `test_integrity_signal_property_keys_match_across_layers` lost the only case it had that reaches both verification layers with the expected hash in hand. They now take a session-scoped `oversize_content` fixture that builds the string from `MAX_SKILL_CONTENT_BYTES + 1`, so it sits one byte past whatever the bound currently is and moving it again cannot silently invert them. Scoped to the session because the string is cap-sized: built and hashed once per run rather than once per test. Reading the constant here is the non-circular direction. `TestPackageExports` spells the literal out on purpose and documents itself as the one place the constants themselves are asserted; these two are about enforcement rather than about the number, so deriving the boundary from the module under test is what keeps them honest instead of making them circular. Verified non-vacuous: with the cap comparison in `verify_raw_skill` temporarily widened so nothing is ever refused, both tests fail. `skills_core.py` is unchanged by this commit. The suite is now fully green on this branch — 1775 passed, 11 skipped, with `ruff check`, `ruff format --check` and `mypy packages/*/src` clean. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills_fs.py | 36 +++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 38649420..ff54d525 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -19,6 +19,7 @@ import launchdarkly_ai_server.safe_fs as safe_fs_module import launchdarkly_ai_server.skills as skills_module +import launchdarkly_ai_server.skills_core as skills_core_module import launchdarkly_ai_server.skills_fs as skills_fs_module from launchdarkly_ai_server import ( InMemorySkillStore, @@ -290,6 +291,30 @@ def _skill( ) +@pytest.fixture(scope="session") +def oversize_content() -> tuple[str, str]: + """Content one byte past the cap, with its hash — ``(content, sha256)``. + + Derived from ``MAX_SKILL_CONTENT_BYTES`` rather than written as a literal, + because a literal is how this went wrong: both consumers were pinned at the + old 64 KiB bound and stayed there when the cap moved to 10 MiB, which put + them *under* the limit and left them asserting that oversize content is + written and reported clean. Derived, they are one byte past whatever the + bound currently is, and moving it again cannot silently invert them. + + The bound's *value* is asserted once, in ``TestPackageExports``, which + spells the literal out on purpose — reading it from the module there would + make that assertion circular. These tests are about enforcement rather than + the number, so reading it here is the non-circular direction. + + Session-scoped: the string is cap-sized, so it is built and hashed once for + the run instead of per test. ``x`` encodes to one byte, so the character + count is the byte count. + """ + content = "x" * (skills_core_module.MAX_SKILL_CONTENT_BYTES + 1) + return content, _hash(content) + + def _manifest_path(root: Path) -> Path: return root / MANIFEST_NAME @@ -909,8 +934,10 @@ async def test_hash_mismatch_aborts_the_write( assert not (root / "a" / "SKILL.md").exists() assert len(recording_emitter.signals(INTEGRITY_SIGNAL)) == 1 - async def test_oversize_skill_aborts_the_write(self, root: Path) -> None: - oversize = "x" * (64 * 1024 + 1) + async def test_oversize_skill_aborts_the_write( + self, root: Path, oversize_content: tuple[str, str] + ) -> None: + oversize, _ = oversize_content report = await write_skills([_skill("a", 1, oversize)], root) assert report.ok is False assert not (root / "a" / "SKILL.md").exists() @@ -2244,7 +2271,7 @@ async def test_throwing_emitter_never_breaks_the_reconcile( assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY async def test_integrity_signal_property_keys_match_across_layers( - self, root: Path, recording_emitter: Any + self, root: Path, recording_emitter: Any, oversize_content: tuple[str, str] ) -> None: """The same defect, caught at either layer, records the same property keys. @@ -2258,8 +2285,7 @@ async def test_integrity_signal_property_keys_match_across_layers( hand throughout. """ skills_module._set_emitter_for_testing(recording_emitter) - oversize = "x" * (64 * 1024 + 1) - content_hash = _hash(oversize) + oversize, content_hash = oversize_content # Layer 1 — the accessor boundary. store = InMemorySkillStore() From 9252fb08f1d6a45bf66f42b9e10e1f85678b5b02 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 14:55:58 -0400 Subject: [PATCH 31/43] fix(client): resolve a missed pin as absent, not as an integrity failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_SkillObjectSet.get` fell through to a key's version-less entry whenever a pinned lookup missed, even when well-formed versions of that key were held. `InMemorySkillStore.get_object` reaches the version-less entry only when the key has no well-formed version at all — the fall-through exists so a malformed object is withheld with a signal rather than reading as absent, not so that it answers for a version that was never delivered. The divergence was caller-visible. Given a key holding both a good version and a malformed sibling, `get_skill_result(key, version=)` reported `absent` through `InMemorySkillStore` and `integrity_failure` through `FDv2SkillStore` — and `integrity_failure` is the one reason the docs tell callers to fail closed on, so a plain version miss could abort a process in production while passing in a test using the in-memory store. The parity test asserted the two agree, but its fixtures held no key with both a well-formed version and a version-less entry, so the divergent quadrant was never exercised. Add that row, and an end-to-end test over the transport. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 26 +++++++------ packages/client/tests/test_skills_fdv2.py | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 71411c8a..7041b52d 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -393,11 +393,11 @@ class _SkillObjectSet: """ Raw skill objects held in memory, keyed by ``(key, version)``. - Lookup semantics are identical to ``InMemorySkillStore``'s, down to the - fall-through to a version-less entry, so that the store a caller configures - cannot change how a pinned reference resolves; ``TestInterfaceParity`` - asserts it. Reimplemented rather than inherited because the transport needs - ``delete`` and the atomic ``replace_with`` a full transfer requires. + Lookup semantics are identical to ``InMemorySkillStore``'s, down to when a + version-less entry is reachable, so the store an application configures + cannot change how a pinned reference resolves. Reimplemented rather than + inherited because the transport needs ``delete`` and the atomic + ``replace_with`` a full transfer requires. An object too malformed to carry a usable version is still held, under its key alone, so verification withholds it with a signal rather than the @@ -437,13 +437,17 @@ def delete(self, tombstone: _Tombstone) -> list[dict[str, Any]]: def get(self, key: str, version: int | None) -> dict[str, Any] | None: held = self._versions.get(key, {}) + if not held: + # Nothing well-formed is filed under this key, so the version-less + # entry is all there is: serve it, and let verification withhold it + # with a signal rather than have it read as simply absent. A pin + # that misses while well-formed versions do exist is a plain miss, + # and answering it with a leftover malformed object would record an + # integrity failure for a skill whose integrity is not in question. + return self._loose.get(key) if version is not None: - # Fall through to the version-less entry so a malformed object - # reaches verification rather than reading as simply absent. - return held.get(version) or self._loose.get(key) - if held: - return held[max(held)] - return self._loose.get(key) + return held.get(version) + return held[max(held)] def snapshot(self) -> dict[str, dict[str, Any]]: """One entry per ``(key, version)``, under keys opaque to the SDK.""" diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index b0c6aa86..f0099db2 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1010,6 +1010,12 @@ class TestInterfaceParity: {"key": "a", "version": 4, "content": "y", "contentHash": _hash("y")}, {"key": "b", "version": 2, "content": "z", "contentHash": _hash("z")}, {"key": "malformed", "version": "not-a-version", "content": "q"}, + # A key holding a well-formed version *and* a version-less entry. The + # quadrant the fixtures above miss: "malformed" has no usable version + # at all, and "a"/"b" have no version-less entry, so neither exercises + # what happens when a pin misses a key that has both. + {"key": "mixed", "version": 2, "content": "m", "contentHash": _hash("m")}, + {"key": "mixed", "version": "not-a-version", "content": "n"}, ] def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: @@ -1033,6 +1039,9 @@ def _both(self) -> tuple[InMemorySkillStore, _SkillObjectSet]: ("missing", 1), ("malformed", None), ("malformed", 7), + ("mixed", None), + ("mixed", 2), + ("mixed", 7), ], ) def test_get_agrees(self, key: str, version: int | None) -> None: @@ -2037,6 +2046,36 @@ async def test_a_pinned_reference_resolves_to_the_pinned_object_version( assert newest is not None assert newest.version == 5 + async def test_a_missed_pin_is_absent_even_beside_a_malformed_sibling( + self, endpoint: Any + ) -> None: + """ + A key can hold a well-formed version and a version-less entry at once — + a malformed object arrives with no version in its wire key, and is held + anyway so verification withholds it with a signal. + + A pin that misses is still a plain miss. Answering it with the + version-less entry would report ``integrity_failure`` for a skill whose + integrity is not in question, and that is the one reason callers are + told to fail closed on. + """ + endpoint.queue_poll( + full_payload( + ("put-object", put_skill(object_version=3)), + ("put-object", put_skill(object_version=None)), + ) + ) + with poll_store(endpoint) as store: + store.wait_for_skills(timeout=5) + await init_client(options={"skillStore": store}, client=object()) + + missed = await get_skill_result("pdf-extraction", version=9) + assert missed.skill is None + assert missed.reason == "absent" + # The well-formed version still resolves, and the malformed sibling + # is still reachable to be withheld when nothing else answers. + assert await get_skill("pdf-extraction", version=3) is not None + async def test_the_payload_version_is_not_resolvable_as_a_skill_version( self, endpoint: Any ) -> None: From ad53fa0a2705db706b4912a6a70fd0bbd43c2a6f Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 12:21:22 -0400 Subject: [PATCH 32/43] fix(client): probe for os.fchmod instead of assuming it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows gained `os.fchmod` in CPython 3.13, and this package supports 3.12. `atomic_write` called it unconditionally — including on the path-fallback branch, which is the only branch Windows ever takes — so every write on Windows/3.12 raised `AttributeError` after the temp file was already open. The documented fallback the module is built around never worked there. CI is ubuntu-only and the existing fallback test runs on POSIX, where `fchmod` exists, so nothing caught it. `_SUPPORTS_FCHMOD` probes for the function and the mode goes on the temp path where it is absent. That concedes nothing on the one platform that takes it: Windows is already on the path-based `lstat` floor, the temp name is unguessable (`mkstemp` with `O_CREAT | O_EXCL`), and read-only is the only bit Windows honours from a POSIX mode. The `elif at_fd is None` guard keeps a hypothetical descriptors-but-no-`fchmod` platform from `chmod`ing a bare name relative to the cwd — it would keep the 0600 the exclusive open already set. Co-Authored-By: Claude Opus 5 --- packages/client/agents.md | 3 ++- .../src/launchdarkly_ai_server/safe_fs.py | 23 ++++++++++++++++++- packages/client/tests/test_safe_fs.py | 15 ++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 82bea289..7c1035d9 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -561,7 +561,8 @@ primitives live in `safe_fs.py`, which knows nothing about skills: platform split once as `if dir_fd is not None` and cannot forget the `os.close`. - `atomic_write` creates the temp file with `O_CREAT | O_EXCL | O_NOFOLLOW` **at** that descriptor (`_mkstemp_at`, since `tempfile` has no `dir_fd` form), `fchmod`s the - descriptor rather than `chmod`ing a path, writes, fsyncs, and renames with + descriptor rather than `chmod`ing a path — probed, because Windows has no `os.fchmod` + before 3.13 and 3.12 is supported — writes, fsyncs, and renames with `os.replace(tmp, name, src_dir_fd=fd, dst_dir_fd=fd)`, then fsyncs the directory so the rename survives a crash. `atomic_write_in` is the same against a directory the caller does not already hold open. `os.replace` is the single rename call site, reached by attribute diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 34bafdec..8fe2b2bc 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -52,6 +52,16 @@ """Mode set explicitly on every written file — never inherited from the umask, and never executable.""" +_SUPPORTS_FCHMOD = hasattr(os, "fchmod") +""" +Whether the mode can be set on the descriptor rather than on a path. + +POSIX always has ``os.fchmod``; Windows only gained it in CPython 3.13, and this +package supports 3.12. Probing for it rather than assuming it is what keeps the +documented Windows fallback a fallback instead of an ``AttributeError`` raised +after the temp file is already open. +""" + SUPPORTS_DIR_FD = os.supports_dir_fd.issuperset( # renameat, openat, unlinkat, fstatat, mkdirat, and unlinkat's AT_REMOVEDIR # form — the six this module and its caller need. @@ -377,7 +387,18 @@ def atomic_write( # fchmod, not chmod: operating on the descriptor cannot be redirected # by anything that swaps the temp path underneath us, and it makes the # mode independent of the process umask (both creation paths open 0600). - os.fchmod(fd, _FILE_MODE) + if _SUPPORTS_FCHMOD: + os.fchmod(fd, _FILE_MODE) + elif at_fd is None: + # Windows before 3.13 has no fchmod — and no ``*at()`` family + # either, so ``temp`` is a full path here and the mode goes on it. + # That concedes nothing this platform was getting: it is already + # on the path-based floor, the temp name is unguessable, and the + # only bit Windows takes from a POSIX mode is read-only. A + # platform with descriptors but no fchmod does not exist; were + # there one it would keep the 0600 the exclusive open already set, + # rather than have a bare name chmoded relative to the cwd. + os.chmod(temp, _FILE_MODE) view = memoryview(data) while view: view = view[os.write(fd, view) :] diff --git a/packages/client/tests/test_safe_fs.py b/packages/client/tests/test_safe_fs.py index b148ff9b..9e622886 100644 --- a/packages/client/tests/test_safe_fs.py +++ b/packages/client/tests/test_safe_fs.py @@ -170,6 +170,21 @@ def test_write_without_a_descriptor_uses_the_path_fallback( assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + def test_the_fallback_works_without_fchmod( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Windows before CPython 3.13 has no ``os.fchmod``, and 3.12 is supported. + + The mode goes on the temp path there instead; what must not happen is an + ``AttributeError`` out of the branch every Windows write takes. + """ + monkeypatch.delattr(os, "fchmod") + monkeypatch.setattr(safe_fs_module, "_SUPPORTS_FCHMOD", False) + atomic_write(tmp_path, "f.txt", b"fallback", dir_fd=None) + assert (tmp_path / "f.txt").read_bytes() == b"fallback" + assert stat.S_IMODE((tmp_path / "f.txt").stat().st_mode) == 0o644 + assert [p.name for p in tmp_path.iterdir()] == ["f.txt"] + def test_write_in_pins_the_directory_itself(self, tmp_path: Path) -> None: atomic_write_in(tmp_path, "f.txt", b"x") assert (tmp_path / "f.txt").read_bytes() == b"x" From 7c070d0acab01294cb1bbdcdca2fe9ac7e4ff133 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 12:23:04 -0400 Subject: [PATCH 33/43] fix(client): close two paths that let the reconcile prune what it should keep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both the same shape: a step that could not be trusted was treated as trustworthy, and prune then read a formerly-managed skill as revoked and deleted it. `_resolve_all` returned `incomplete=False` unconditionally. Every per-object failure it can attribute to a key keeps that key in the requested set, which is what holds prune off the copy on disk — but a failure with no usable key on either side (the object's own `key` field, or the store's `all_objects` key) is recorded run-level, and has no key to do that with. The shipped `InMemorySkillStore` keys `all_objects` as `:`, which is never a valid skill key, so one object arriving with a mangled `key` field was enough: the reconcile reported the withholding and then deleted the last known-good copy in the same run. It now reports the run incomplete, which suppresses pruning wholesale. The manifest gate rejected only `manifestVersion` above the version this release writes. No release ever wrote a version below 1, so `0` and negatives are not older schemas we can still read — they are schemas that never existed, and reading one ran the destructive steps against entries of unknown shape. Verified before the fix: `manifestVersion: 0` pruned a managed file and reported `ok` true. The gate is now bounded at both ends, kept as a range so `MANIFEST_VERSION`'s "highest it can read" still holds when version 2 lands. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fs.py | 19 +++++- packages/client/tests/test_skills_fs.py | 58 +++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 2c098995..460fa40f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -517,7 +517,9 @@ def _pending_for_raw(object_key: str, raw: Any) -> _PendingWrite: key = raw_key if is_valid_skill_key(raw_key) else object_key if not is_valid_skill_key(key): # Neither key is usable, so this failure cannot be attributed to a skill - # — the run-level sentinel is the honest report. + # — the run-level sentinel is the honest report. ``_resolve_all`` reads + # that sentinel back as an incomplete run, because a failure with no key + # cannot protect its copy on disk the way the branch below does. return _PendingWrite( key=_RUN_LEVEL_KEY, error="the skill store served an object under an invalid key; " @@ -555,7 +557,15 @@ def _resolve_all( len(requests), sum(1 for request in requests if request.skill is not None), ) - return requests, False + # A withholding that could not be attributed to a key leaves the run + # incomplete. Every other failure keeps its key in the requested set, which + # is what holds prune off the copy on disk; a run-level failure has no key to + # do that with, so suppressing prune wholesale is the only thing left that + # stops an unreadable object reading as a revocation. + unattributed = any( + request.skill is None and request.key == _RUN_LEVEL_KEY for request in requests + ) + return requests, unattributed # ------------------------------------------------------------------------- @@ -686,10 +696,13 @@ def _load_manifest( ) version = data.get("manifestVersion") + # Bounded at both ends. No release ever wrote a version below 1, so 0 or a + # negative is not an older schema this release could still read — it is a + # schema that never existed, and acting on its entries would be a guess. if ( not isinstance(version, int) or isinstance(version, bool) - or version > MANIFEST_VERSION + or not 1 <= version <= MANIFEST_VERSION ): return {}, ( f"the skills manifest {MANIFEST_FILENAME} declares manifestVersion " diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index ff54d525..8db3a119 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -445,6 +445,33 @@ async def test_entry_paths_are_forward_slash_relative(self, root: Path) -> None: assert "\\" not in keys[0] assert not keys[0].startswith("/") + @pytest.mark.parametrize("declared", [0, -1]) + async def test_manifest_version_below_one_is_corrupt( + self, root: Path, declared: int + ) -> None: + """The version gate is bounded below as well as above. + + No release ever wrote a version under 1, so one is not an older schema + this release can still read. Treating it as readable would run the + destructive steps against entries of unknown shape. + """ + target = _place_managed(root, "a", SKILL_BODY) + _write_manifest( + root, + { + "manifestVersion": declared, + "entries": {"a/SKILL.md": _entry("a", 1, SKILL_BODY)}, + }, + ) + + report = await write_skills([], root) + + assert report.ok is False + assert any("manifestVersion" in m for m in _error_messages(report)) + assert target.exists(), "pruned against a manifest schema that never existed" + # The manifest itself is left alone, as with any other corruption. + assert _read_manifest(root)["manifestVersion"] == declared + async def test_unknown_fields_are_preserved_on_rewrite(self, root: Path) -> None: entry = _entry("a", 1, SKILL_BODY) entry["futureEntryField"] = "keep-me" @@ -901,6 +928,37 @@ def all_objects(self, kind: str) -> dict[str, Any]: assert _actions_by_key(report)["requested-key"].action == "error" assert _actions_by_key(report)["other-key"].action == "skipped_current" + async def test_an_unattributable_failure_never_prunes(self, root: Path) -> None: + """A withholding with no usable key at all still must not prune. + + The sibling case above recovers the object's own key and keeps it in the + requested set. When neither the object's key nor the store's is usable + the failure is run-level, so there is no key to hold the on-disk copy + with — reporting the run incomplete is the only thing left that stops + prune reading an unreadable object as a revocation. + """ + existing = _place_managed(root, "pdf-extraction", SKILL_BODY) + # The shipped store keys ``all_objects`` as ":", which is + # never a valid skill key, so a missing 'key' field leaves no fallback. + skills_module._set_store( + InMemorySkillStore( + { + "pdf-extraction:1": { + "version": 1, + "content": SKILL_BODY, + "contentHash": _hash(SKILL_BODY), + } + } + ) + ) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a.action for a in report.actions] == ["error"] + assert "pdf-extraction/SKILL.md" in _read_manifest(root)["entries"] + async def test_unavailable_run_does_not_corrupt_manifest(self, root: Path) -> None: _place_managed(root, "a", SKILL_BODY) before = _read_manifest(root) From 098455f6a7de60ae59955ebbc1a298e3558ec754 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 14:53:53 -0400 Subject: [PATCH 34/43] fix(client): stop reporting a withholding for a key that resolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `newest_by_key` kept every object too malformed to carry a usable key and version, so verification rather than the collapse would withhold it. That is right when nothing else resolves the key — the key stays in the requested set and prune leaves the copy on disk alone — but wrong when another version of the same key resolved anyway. In that case `write_skills("*")` reported both a `written` and an `error` action for one key, flipping `report.ok` to False for a skill that is correctly on disk, and saying the copy there "was left alone" when the same run had just written it. `all_skills()` logged a withholding count for the same key. Drop a malformed object only when its own skill key resolved from another version; keep it otherwise, which is the case the protection exists for. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_core.py | 22 +++++++--- packages/client/tests/test_skills.py | 24 +++++++++++ packages/client/tests/test_skills_fs.py | 40 +++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 3215f394..c51f6382 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -679,10 +679,13 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: The store key is carried through rather than discarded because the reconcile attributes a failure to it when the object's own key is unusable. - Objects too malformed to carry a usable key and version are **kept**, not - dropped, so verification is what withholds them: a silently dropped object - falls out of the requested set, and prune would then delete the last - known-good copy already on disk. + An object too malformed to carry a usable key and version is **kept**, so + verification is what withholds it: a silently dropped object falls out of + the requested set, and prune would then delete the last known-good copy + already on disk. The exception is an object whose skill key resolved anyway + from another version — there the resolved object already holds the key in + the requested set, so keeping the malformed one would only report a + withholding for a key that in fact resolved. """ best: dict[str, tuple[str, Any]] = {} unusable: list[tuple[str, Any]] = [] @@ -695,7 +698,16 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: held = best.get(skill_key) if held is None or version > held[1]["version"]: best[skill_key] = (object_key, raw) - return list(best.values()) + unusable + withheld = [ + (object_key, raw) + for object_key, raw in unusable + # ``is_valid_skill_key`` first: an unhashable key cannot be looked up. + if not ( + is_valid_skill_key(raw.get("key") if isinstance(raw, dict) else None) + and raw["key"] in best + ) + ] + return list(best.values()) + withheld # --------------------------------------------------------------------------- diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index db709ab6..466efbf0 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1414,6 +1414,30 @@ async def test_get_skills_warns_once_per_batch( warnings = [r for r in caplog.records if r.levelname == "WARNING"] assert len(warnings) == 1 + async def test_a_key_that_resolved_is_not_also_reported_withheld( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """ + A store may hold a malformed object beside a well-formed version of the + same key. The well-formed one resolves, so the key is not withheld and + must not be counted as though it were. + """ + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="a", version="not-a-version")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + skills = await all_skills() + assert [s.key for s in skills] == ["a"] + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + async def test_a_key_with_no_usable_version_is_still_withheld( + self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture + ) -> None: + """The converse: nothing resolved the key, so the withholding stands.""" + store.put(make_raw_skill(key="a", version="not-a-version")) + with caplog.at_level("WARNING", logger="launchdarkly_ai_server.skills_core"): + assert await all_skills() == [] + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + async def test_a_fully_resolved_run_is_silent( self, store: Any, make_raw_skill: Any, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 8db3a119..ab353449 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -537,6 +537,46 @@ async def test_prune_removes_formerly_managed_skill(self, root: Path) -> None: assert not (root / "gone").exists() assert _read_manifest(root)["entries"] == {} + async def test_star_does_not_report_an_error_for_a_key_that_wrote( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """A malformed object beside a good version of the same key. + + The good version resolves and materializes, so the run succeeded for + that key. Reporting the malformed sibling as well would flip + ``report.ok`` to ``False`` for a skill that is correctly on disk, and + say the copy there "was left alone" when this run had just written it. + """ + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="a", version="not-a-version")) + + report = await write_skills("*", root) + + assert report.ok is True + assert [(a.key, a.action) for a in report.actions] == [("a", "written")] + assert (root / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + async def test_star_still_reports_a_key_nothing_could_resolve( + self, root: Path, store: InMemorySkillStore, make_raw_skill: Any + ) -> None: + """The converse, and the reason the malformed object is kept at all. + + No version of ``b`` is usable, so it stays in the requested set: the + failure is reported, and prune leaves any copy already on disk alone + rather than reading the key as revoked. + """ + existing = _place_managed(root, "b", SKILL_BODY) + store.put(make_raw_skill(key="a", version=1)) + store.put(make_raw_skill(key="b", version="not-a-version")) + + report = await write_skills("*", root) + + assert report.ok is False + assert _actions_by_key(report)["a"].action == "written" + assert _actions_by_key(report)["b"].action == "error" + assert [a for a in report.actions if a.action == "removed"] == [] + assert existing.read_text(encoding="utf-8") == SKILL_BODY + async def test_prune_false_keeps_the_file(self, root: Path) -> None: target = _place_managed(root, "gone", SKILL_BODY) From 5a896cef51041ab95ab03515b0f0b075b05d98a2 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 14:58:56 -0400 Subject: [PATCH 35/43] fix(client): do not prune when the store has not received its initial data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Through the `SkillStore` interface, a store still waiting for its first payload and an environment that holds no skills give the same answer: an empty one. `write_skills("*")` read that as every skill having been revoked, so a reconcile racing a slow boot deleted every file a previous run had written and reported `ok == True`. `FDv2SkillStore` holds nothing until its first payload commits and `all_objects` returns `{}` rather than raising, so this was reachable with no fault anywhere — and `wait_for_skills` returning False was enough to trigger it, which the README example did not check. Add an optional `is_initialized()` to the store interface, probed the same way `add_listener` already is: absent means initialized, which is correct for a hand-populated store, and a probe that raises counts as not initialized. The check lives in `_available_store`, the single gate that already decides whether pruning may run, so retrieval reports itself unavailable exactly as it does for a raising store. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 16 ++- packages/client/agents.md | 22 ++++- .../src/launchdarkly_ai_server/skills_core.py | 47 ++++++++- .../src/launchdarkly_ai_server/skills_fdv2.py | 17 ++++ .../src/launchdarkly_ai_server/skills_fs.py | 16 +++ packages/client/tests/test_skills_fdv2.py | 26 +++++ packages/client/tests/test_skills_fs.py | 99 +++++++++++++++++++ 7 files changed, 235 insertions(+), 8 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 2bc83b7b..4503f53c 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -506,7 +506,9 @@ import os from launchdarkly_ai_server import FDv2SkillStore, init_client, watch_skills store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() -store.wait_for_skills(timeout=10) +if not store.wait_for_skills(timeout=10): + # No payload arrived. Reconciling now would find an empty store; see below. + print(f"skill delivery has not answered yet: {store.failed or 'still waiting'}") await init_client(options={"skillStore": store}) # Materialize now, and re-materialize whenever delivery changes. @@ -518,6 +520,14 @@ finally: store.close() ``` +**A reconcile that runs before delivery answers does not prune.** Through the `SkillStore` +interface, a store still waiting for its first payload and an environment that holds no +skills give the same answer — an empty one — and `write_skills("*")` would otherwise read +that as every skill having been revoked and delete the files it wrote on a previous run. +`FDv2SkillStore` reports readiness through the optional `is_initialized()`, so a reconcile +against a store that has not received a payload reports the retrieval unavailable and leaves +everything on disk alone. `report.ok` is `False` in that case, and the error names it. + **Nothing above the store changes.** The accessors, verification, and `write_skills` see raw objects through the `SkillStore` interface and cannot tell which store produced them. @@ -566,9 +576,9 @@ Windows. | `get_skills(refs)` | Batch form. Accepts `SkillReference` values and bare key strings (string = latest). Results follow input order; missing or unverifiable entries are omitted. | | `all_skills()` | Every verified skill the store holds, one per key at its newest version. | | `write_skills(skills, root, *, prune=True, timeout=10.0, on_unavailable="keep")` | Materialize skills under `root`, returning a `ReconcileReport`. `prune` removes formerly-managed skills no longer requested. `on_unavailable="raise"` raises instead of reporting when content cannot be retrieved. Raises `ValueError` for an unusable root, a negative `timeout`, or an unrecognised `on_unavailable`. **Performs synchronous filesystem I/O — see the note below.** | -| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `add_listener(kind, fn)` / `remove_listener(kind, fn)`. | +| `SkillStore` | The structural interface content arrives through: `get_object(kind, key, version=None)`, `all_objects(kind)`, optional `is_initialized()`, `add_listener(kind, fn)` / `remove_listener(kind, fn)`. A store without `is_initialized()` is treated as initialized. | | `InMemorySkillStore(objects=None)` | A dict-backed store with `put(raw)`, for local development and testing. Holds several versions of a key. | -| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | +| `FDv2SkillStore(sdk_key, *, base_uri=…, mode="stream", …)` | The delivery transport: a store fed by LaunchDarkly over the SDK-facing FDv2 channel. `start()`, `wait_for_skills(timeout)`, `is_initialized()`, `close()`, `diagnostics`, `failed`; also a context manager. **Server-side only** — a mobile key or client-side environment ID raises. See *Receiving skills from LaunchDarkly* above. | | `watch_skills(skills, root, …)` | `write_skills` plus a re-reconcile on every delivery change. Returns `(initial report, SkillWatcher)`; close the watcher when done. Revocation then takes effect within `debounce` of arriving rather than at the next restart. | | `StoreDiagnostics` | What the transport has seen: `payloads_transferred`, `skill_objects_received`, `objects_ignored`, `objects_revoked`, `hashless_objects`, `connection_failures`, `last_error`. | diff --git a/packages/client/agents.md b/packages/client/agents.md index 7c1035d9..49375164 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -211,7 +211,8 @@ Three layers, in increasing order of blast radius: ### The store seam, and why version is part of the lookup `SkillStore` is `get_object(kind, key, version=None)`, `all_objects(kind)`, and the optional -pair `add_listener(kind, fn)` / `remove_listener(kind, fn)`. Version is part of the **lookup +`is_initialized()` plus the optional pair `add_listener(kind, fn)` / +`remove_listener(kind, fn)`. Version is part of the **lookup identity**, not a filter applied to the answer, and that is load-bearing: a delivery payload carries the newest version of every skill *plus* every version any variation currently pins, so two versions of one key coexist routinely. A store keyed by key alone would answer a pinned reference with the newest @@ -227,7 +228,24 @@ SDK. Do not parse them and do not assume one per skill key; identity is read off object's own `key` and `version`, which are revalidated anyway. `newest_by_key` is the one place that collapses the result to one object per key, because both whole-store consumers need it — `all_skills`, since a list holding two versions of one key is not a set -of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. +of skills, and the `"*"` reconcile, since `//SKILL.md` is a single path. It keeps +an object too malformed to carry a usable key and version, so verification is what withholds +it and the key stays in the requested set where prune cannot touch its on-disk copy — unless +another version resolved that key anyway, in which case keeping it would only report a +withholding for a key that resolved. + +### `is_initialized()` is what stands between a slow boot and deleting a customer's files + +Through the store interface, "this environment holds no skills" and "delivery has not +answered yet" are the same empty answer, and `write_skills("*")` reads the first as every +skill having been revoked. So `_available_store` — the single gate that sets `unavailable` +and therefore suppresses pruning — consults `store_is_initialized`, and a store that has not +received its initial data blocks retrieval instead of authorizing a prune. A store that does +not implement the probe is treated as initialized, which is right for one populated by hand; +a probe that *raises* counts as not initialized, because a store that cannot say whether it +is ready is not one to delete on. Keep this check in that one gate: maintained in two places, +a condition added to one and not the other does not merely produce a wrong message — it +deletes the user's files. ### The delivery transport, and the one field that will bite you diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index c51f6382..6577c597 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -166,9 +166,9 @@ class SkillStore(Protocol): Duck-typed on purpose, mirroring how the LaunchDarkly client interface works in this package: pass any object carrying these methods. - ``add_listener(kind, fn)`` and ``remove_listener(kind, fn)`` are part of the - interface but **optional**, which is why they are deliberately not declared - here: a Protocol member is required for structural compatibility, so declaring + ``is_initialized()``, ``add_listener(kind, fn)`` and + ``remove_listener(kind, fn)`` are part of the interface but **optional**, + which is why they are deliberately not declared here: a Protocol member is required for structural compatibility, so declaring them would reject every store that does not implement them. Nothing in this module calls either — they exist for the delivery transport to push updates through, and for a consumer such as ``watch_skills`` to stop receiving them. @@ -178,6 +178,14 @@ class SkillStore(Protocol): the store does. ``remove_listener`` removes one occurrence of *fn* under *kind* and is a no-op when *fn* is not registered. + ``is_initialized()`` reports whether the store has received its initial data + — for a delivery transport, whether a payload has arrived yet. A store that + does not implement it is treated as initialized, which is correct for one + that is populated by hand. It matters because "the store holds nothing" and + "the store has not heard yet" are the same answer through ``all_objects``, + and ``write_skills("*")`` reads the first as "every skill was revoked": see + ``store_is_initialized``. + The raw objects a store serves are wire-shaped, with camelCase field names identical across language implementations:: @@ -281,6 +289,39 @@ def require_store() -> SkillStore: return store +def store_is_initialized(store: Any) -> bool: + """ + Whether *store* has received its initial data. + + ``True`` for a store that does not implement the optional + ``is_initialized()``, since a hand-populated store is never waiting for + anything. Probed rather than declared on the Protocol for the reason + ``SkillStore`` gives: a declared member would reject every store without it. + + This is what keeps ``write_skills("*")`` from reading a store that has not + yet received a payload as an environment whose every skill was revoked. + Retrieval through such a store is reported unavailable, which suppresses + pruning — the same treatment a raising store gets, and for the same reason: + deleting a customer's files because content could not be retrieved would + turn a slow boot into data loss. + + A probe that raises counts as not initialized. A store that cannot answer + whether it is ready is not one to authorize deletions on. + """ + probe = getattr(store, "is_initialized", None) + if not callable(probe): + return True + try: + return bool(probe()) + except Exception: + logger.warning( + "The skill store's is_initialized() raised; treating the store as " + "not yet initialized", + exc_info=True, + ) + return False + + def emit(signal: str, properties: dict[str, Any]) -> None: """ Records one signal. Never raises into the calling operation — a broken diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 7041b52d..f8812b33 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1385,6 +1385,23 @@ def _end_delivery(self) -> None: self._delivery_ended.set() self._released.set() + def is_initialized(self) -> bool: + """ + Whether a payload has arrived, so reads reflect delivery rather than an + empty store still waiting for its first one. + + The optional half of the ``SkillStore`` interface, and the same fact + ``wait_for_skills`` returns — without the wait. ``write_skills("*")`` + consults it so a reconcile that runs before delivery reports the + retrieval unavailable rather than pruning every managed skill as though + the environment had revoked it. + + Stays ``True`` once a payload has arrived, including after ``close``: a + closed store still answers from what it received, and a later + reconcile against that content is a reconcile against real delivery. + """ + return self._first_payload.is_set() + @property def failed(self) -> str | None: """Why delivery stopped for good, or ``None`` while it is running.""" diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 460fa40f..9f7f170b 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -57,6 +57,7 @@ record_revoked, reference_target, resolve_from_store, + store_is_initialized, verified_bytes, verify_raw_skill, ) @@ -411,6 +412,21 @@ def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBl store = get_store() if store is None: return _RetrievalBlocked(_unavailable(NO_STORE_MESSAGE)) + if not store_is_initialized(store): + # A store still waiting for its first delivery answers every read with + # "nothing", which is indistinguishable from an environment that holds + # no skills — and the "*" form reads that as every skill having been + # revoked. Blocking here reports the run incomplete, which is what + # suppresses the prune. + return _RetrievalBlocked( + _unavailable( + "the skill store has not received its initial data, so " + f"{subject} could not be retrieved and nothing on disk was " + "changed. Wait for delivery before reconciling: " + "FDv2SkillStore.wait_for_skills(timeout) returns True once the " + "first payload has arrived." + ) + ) return store diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index f0099db2..cb99a8d4 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -2528,6 +2528,32 @@ def test_close_releases_a_waiter_rather_than_leaving_it_parked(self) -> None: assert time.monotonic() - started < 2.0 assert answers == [False] + def test_is_initialized_tracks_the_first_payload(self) -> None: + """The probe ``write_skills("*")`` reads to decide whether it may prune. + + Before the first payload, an empty store and an environment with no + skills are the same answer through ``all_objects``; this is what tells + them apart. + """ + store = stream_store(_requester=_SilentStreamRequester()) + try: + assert store.is_initialized() is False + store.start() + assert store.wait_for_skills(timeout=0.2) is False + assert store.is_initialized() is False + finally: + store.close() + + delivering = stream_store(_requester=_RecyclingRequester()) + try: + delivering.start() + assert delivering.wait_for_skills(timeout=5) is True + assert delivering.is_initialized() is True + finally: + delivering.close() + # Content outlives the connection, so the fact about it does too. + assert delivering.is_initialized() is True + def test_a_payload_already_held_still_answers_true_after_close(self) -> None: # ``close`` does not drop content, so it must not turn the answer about # that content into a lie either. diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index ab353449..742d9fb6 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -748,6 +748,105 @@ async def test_bare_string_writes_nothing(self, root: Path) -> None: assert list(root.iterdir()) == [] +class TestUninitializedStore: + """A store that has not received its initial data must not authorize a prune. + + Through ``all_objects`` there is no difference between "this environment + holds no skills" and "delivery has not answered yet": both are an empty + result. ``write_skills("*")`` reads the first as every skill having been + revoked, so without the optional ``is_initialized()`` probe a reconcile + racing a slow boot deletes every managed file and reports success. + """ + + class _Waiting: + """A delivery store whose first payload has not arrived.""" + + def __init__(self, initialized: bool = False) -> None: + self._initialized = initialized + + def is_initialized(self) -> bool: + return self._initialized + + def get_object( + self, kind: str, key: str, version: int | None = None + ) -> dict[str, Any] | None: + return None + + def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: + return {} + + async def test_star_does_not_prune_before_the_first_payload( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + existing = _place_managed(root, "a", SKILL_BODY) + spy = _UnlinkSpy().install(monkeypatch) + skills_module._set_store(self._Waiting()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.read_text(encoding="utf-8") == SKILL_BODY + assert [a for a in report.actions if a.action == "removed"] == [] + assert spy.targets == [] + # The entry survives, so a later reconcile still knows it owns the file. + assert "a/SKILL.md" in _read_manifest(root)["entries"] + assert any("initial data" in m for m in _error_messages(report)) + + async def test_star_raises_in_raise_mode_before_the_first_payload( + self, root: Path + ) -> None: + skills_module._set_store(self._Waiting()) + with pytest.raises(RuntimeError, match=r"initial data"): + await write_skills("*", root, on_unavailable="raise") + + async def test_an_initialized_store_prunes_as_before( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The positive control: the probe must not disable pruning outright.""" + _place_managed(root, "a", SKILL_BODY) + skills_module._set_store(self._Waiting(initialized=True)) + + report = await write_skills("*", root) + + assert report.ok is True + assert _actions_by_key(report)["a"].action == "removed" + assert not (root / "a").exists() + + async def test_a_store_without_the_probe_prunes_as_before( + self, root: Path, store: InMemorySkillStore + ) -> None: + """``is_initialized()`` is optional; absent means initialized. + + A hand-populated store is never waiting for anything, so requiring the + probe would break every ``InMemorySkillStore`` caller. + """ + assert not hasattr(store, "is_initialized") + _place_managed(root, "a", SKILL_BODY) + + report = await write_skills("*", root) + + assert report.ok is True + assert _actions_by_key(report)["a"].action == "removed" + + async def test_a_probe_that_raises_counts_as_uninitialized( + self, root: Path + ) -> None: + """A store that cannot say whether it is ready does not get to delete.""" + + class Exploding(TestUninitializedStore._Waiting): + def is_initialized(self) -> bool: + raise RuntimeError("cannot tell") + + existing = _place_managed(root, "a", SKILL_BODY) + skills_module._set_store(Exploding()) + + report = await write_skills("*", root) + + assert report.ok is False + assert existing.exists() + assert [a for a in report.actions if a.action == "removed"] == [] + + class TestResilience: """Unavailable retrieval and timeout.""" From b0ac0e33e7407e4427f7dcc88213ce6b7a8e4d05 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 15:02:20 -0400 Subject: [PATCH 36/43] fix(client): bound the manifest read, and two smaller reconcile fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The manifest read is bounded. It was the one unbounded read in the reconcile, on a file that lives in a directory the SDK does not own exclusively. With every read now capped, `_read_regular_file` loses its unbounded branch. - `FDv2SkillStore.failed` is cleared when the store is started again. It reports why delivery stopped *for good*, which is not true of a store that is delivering again, so a recovered store no longer reports a stale reason. - `ReconcileActionKind` documents each outcome, including that `removed` is reported whether or not this run was the one that deleted the file — a formerly managed file already gone by hand reaches the same end state. The `manifestVersion` lower bound this change originally carried landed earlier; only its comment, which also explains the upper bound, is kept here. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 6 ++ .../src/launchdarkly_ai_server/skills_fs.py | 55 +++++++++++++------ .../src/launchdarkly_ai_server/types.py | 12 +++- packages/client/tests/test_skills_fdv2.py | 19 +++++++ packages/client/tests/test_skills_fs.py | 53 ++++++++++++++++++ 5 files changed, 127 insertions(+), 18 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index f8812b33..702ca30f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1321,8 +1321,14 @@ def _rearm_waiters(self) -> None: Re-arms ``wait_for_skills`` for a store being started again after a ``close``. A payload already held stays an answer; an ended delivery does not, or the next waiter would be released before it began. + + A terminal ``failed`` reason is dropped for the same reason: it says why + delivery stopped for good, and delivery is about to run again. Leaving + it would have a healthy store reporting a failure it has recovered from. + Called with the lock held, which is what ``failed`` reads under. """ self._delivery_ended.clear() + self._failed_reason = None if not self._first_payload.is_set(): self._released.clear() diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 9f7f170b..4d1a74d0 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -82,6 +82,19 @@ MANIFEST_VERSION = 1 """Manifest schema version this release writes, and the highest it can read.""" +_MAX_MANIFEST_BYTES = 8 * 1024 * 1024 +""" +Hard cap on the manifest, so a reconcile cannot be made to read an arbitrary +amount into memory. + +Set far above any real manifest: an entry is a path, a key, a version, a digest +and a timestamp, so even tens of thousands of skills land a couple of orders of +magnitude below this. Anything larger is treated as corruption, which is what +every other unreadable manifest is — the file lives in a directory the SDK does +not own exclusively, and a reconcile must not be the thing that exhausts the +process. +""" + SKILL_FILENAME = "SKILL.md" """The single file each skill materializes to, under ``//``.""" @@ -682,7 +695,7 @@ def _load_manifest( try: raw = _read_regular_file( MANIFEST_FILENAME if root_fd is not None else root / MANIFEST_FILENAME, - max_bytes=None, + max_bytes=_MAX_MANIFEST_BYTES, dir_fd=root_fd, ) except FileNotFoundError: @@ -690,6 +703,14 @@ def _load_manifest( except OSError as exc: return {}, f"the skills manifest {MANIFEST_FILENAME} could not be read: {exc}" + # ``_read_regular_file`` stops one byte past the cap, which is the byte that + # proves the overage without reading the rest of the file. + if len(raw) > _MAX_MANIFEST_BYTES: + return {}, ( + f"the skills manifest {MANIFEST_FILENAME} is larger than the " + f"{_MAX_MANIFEST_BYTES} byte cap; refusing every destructive action" + ) + try: text = raw.decode("utf-8") except UnicodeDecodeError as exc: @@ -712,9 +733,11 @@ def _load_manifest( ) version = data.get("manifestVersion") - # Bounded at both ends. No release ever wrote a version below 1, so 0 or a - # negative is not an older schema this release could still read — it is a - # schema that never existed, and acting on its entries would be a guess. + # Bounded at both ends. Above, because a manifest from a future release may + # record fields whose meaning this one would guess at. Below, because 1 is + # the first version ever written, so 0 or a negative is not an older schema + # this release could still read — it is a schema that never existed, and + # acting on its entries would be a guess. if ( not isinstance(version, int) or isinstance(version, bool) @@ -927,7 +950,7 @@ def failed(message: str) -> ReconcileAction: def _read_regular_file( - target: Path | str, *, max_bytes: int | None, dir_fd: int | None = None + target: Path | str, *, max_bytes: int, dir_fd: int | None = None ) -> bytes: """ Reads *target*, refusing anything that is not a regular file. @@ -951,13 +974,12 @@ def _read_regular_file( ``/SKILL.md``, so covering it the same way needs a descriptor for the skill directory rather than for the root. - ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes: the comparison - consumer only ever hashes the result, and anything longer than the resolved - content cannot match it, so the one extra byte is enough to prove - inequality — which is what keeps a foreign file of arbitrary size from being - pulled into memory now that adoption reads files the manifest does not - list. ``None`` reads to EOF, for the manifest, whose length no caller can - predict and which is parsed rather than compared. + ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes, so no read here + can be made to pull an arbitrary file into memory. The extra byte is what + lets a caller tell "at the cap" from "over it" without reading the rest: + the comparison consumer hashes the result and anything longer than the + resolved content cannot match it, and the manifest reader reports the + overage as corruption. """ flags = ( os.O_RDONLY @@ -970,14 +992,13 @@ def _read_regular_file( if not stat.S_ISREG(os.fstat(fd).st_mode): raise OSError("the target file is not a regular file") chunks: list[bytes] = [] - remaining = None if max_bytes is None else max_bytes + 1 - while remaining is None or remaining > 0: - chunk = os.read(fd, 65536 if remaining is None else min(remaining, 65536)) + remaining = max_bytes + 1 + while remaining > 0: + chunk = os.read(fd, min(remaining, 65536)) if not chunk: break chunks.append(chunk) - if remaining is not None: - remaining -= len(chunk) + remaining -= len(chunk) return b"".join(chunks) finally: os.close(fd) diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 4bbd4782..291b457c 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -518,7 +518,17 @@ class SkillOutcome: ReconcileActionKind = Literal[ "written", "updated", "skipped_current", "removed", "error" ] -"""The closed set of outcomes ``write_skills`` reports.""" +""" +The closed set of outcomes ``write_skills`` reports. + +- ``written`` — the file did not exist and now holds the resolved content. +- ``updated`` — a managed file held different bytes and was overwritten. +- ``skipped_current`` — the bytes on disk already are the resolved content. +- ``removed`` — the skill is no longer managed and is not on disk. Reported + whether or not this run was the one that deleted the file, since a formerly + managed file a caller had already removed by hand reaches the same end state. +- ``error`` — the outcome was refused or failed; see ``ReconcileAction.error``. +""" @dataclass(frozen=True) diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index cb99a8d4..1277fdf3 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1513,6 +1513,25 @@ def test_a_403_stops_delivery_and_explains_why( assert "opt-in" in store.failed assert any("opt-in" in r.getMessage() for r in caplog.records) + def test_a_restarted_store_does_not_report_the_old_failure( + self, endpoint: Any + ) -> None: + """``failed`` says why delivery stopped *for good*. + + A store started again is delivering, so the terminal reason from the + previous run is no longer true of it. Leaving it would have a healthy + store reporting a failure it has recovered from. + """ + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert "401" in store.failed + + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + def test_a_401_stops_delivery(self, endpoint: Any) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 742d9fb6..7490aab0 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -2019,6 +2019,13 @@ def _live_entries() -> dict[str, Any]: "version_not_int_live_entries", {"manifestVersion": "1", "entries": _live_entries()}, ), + # Bounded below as well as above: 1 is the first version ever written, so a + # manifest declaring 0 or a negative is not one this SDK produced. + ("version_zero", {"manifestVersion": 0, "entries": {}}), + ( + "version_negative_live_entries", + {"manifestVersion": -1, "entries": _live_entries()}, + ), ] LIVE_ENTRY_MANIFESTS: list[tuple[str, Any]] = [ @@ -2026,6 +2033,52 @@ def _live_entries() -> dict[str, Any]: ] +class TestOversizeManifest: + """A manifest past the read cap is corruption, not a reason to allocate. + + Every other read in the reconcile is bounded. The manifest is a plain file + in a directory the SDK does not own exclusively, so an unbounded read of it + is a way to have a reconcile exhaust the process. + """ + + async def test_a_manifest_over_the_cap_is_refused_non_destructively( + self, root: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = _place_managed(root, "a", SKILL_BODY) + # Derived from the cap rather than written as a literal, so raising the + # bound cannot leave this test passing against a read it no longer bounds. + filler = "x" * (skills_fs_module._MAX_MANIFEST_BYTES + 1) + _write_manifest(root, {"manifestVersion": 1, "entries": {}, "pad": filler}) + spy = _UnlinkSpy().install(monkeypatch) + + report = await write_skills([], root) + + assert report.ok is False + assert any("cap" in m for m in _error_messages(report)) + # Non-destructive on both counts: nothing removed, and the manifest the + # SDK could not read is left for an operator rather than overwritten. + assert spy.targets == [] + assert target.read_text(encoding="utf-8") == SKILL_BODY + assert "pad" in json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + async def test_a_manifest_at_the_cap_is_still_read(self, root: Path) -> None: + """The positive control — the bound is a cap, not an off-by-one refusal.""" + manifest: dict[str, Any] = {"manifestVersion": 1, "entries": {}} + # Pad to exactly the cap, accounting for the rest of the document. + overhead = len(json.dumps({**manifest, "pad": ""}).encode("utf-8")) + manifest["pad"] = "x" * (skills_fs_module._MAX_MANIFEST_BYTES - overhead) + _write_manifest(root, manifest) + assert ( + _manifest_path(root).stat().st_size == skills_fs_module._MAX_MANIFEST_BYTES + ) + + report = await write_skills([], root) + + assert report.ok is True + # Read, so the unknown field round-trips as any other future field does. + assert "pad" in json.loads(_manifest_path(root).read_text(encoding="utf-8")) + + class TestCorruptManifest: """Corrupt manifest fails closed, non-destructively.""" From a6eeae9aae05d558856d4fbd656497d01c49a88c Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 15:11:23 -0400 Subject: [PATCH 37/43] docs(client): sweep the skills comments for a public SDK audience MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These modules ship in a public package and customers read them to learn how to consume the SDK, so the comments should explain the code, not the project. Removed: references to CI runners and sibling SDK repositories; roadmap and prioritization notes; cross-language parity used as justification (the constraints those defended, such as `sort_keys` on the integrity record, are kept and restated in terms of what they guarantee); pointers to `agents.md`, which customers do not have; a test class name cited as a guarantee; notes about what tests intercept or inject; and a local "verified on CPython 3.12" aside. Also removed the development history — "this used to be", "no longer does", "before this check existed" — which a reader who never saw the earlier version cannot use. Replaced "seam" with "interface" throughout, including in section headings, and the first-person plural with "the SDK" or "the application". Compressed the essay-length rationale on internal helpers and module-private constants down to the invariant plus its reason. The public API reference — `SkillStore`, `write_skills`, `FDv2SkillStore`, the accessors and the exported types — is kept thorough, since that is what a consumer reads first; the `FDv2SkillStore` example now checks `wait_for_skills`, matching the README. No behavior change. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/safe_fs.py | 106 +++--- .../src/launchdarkly_ai_server/skills.py | 65 ++-- .../src/launchdarkly_ai_server/skills_core.py | 310 +++++++----------- .../src/launchdarkly_ai_server/skills_fdv2.py | 74 ++--- .../src/launchdarkly_ai_server/skills_fs.py | 262 +++++++-------- .../launchdarkly_ai_server/skills_watch.py | 6 +- .../src/launchdarkly_ai_server/types.py | 12 +- 7 files changed, 346 insertions(+), 489 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 8fe2b2bc..f217ee57 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -1,39 +1,24 @@ """ Descriptor-pinned filesystem primitives. -Split out because none of this knows what a skill is: it is the "write a file -under a directory an attacker may be racing you for" problem, solved once. -``skills_fs.py`` is the only caller today. - -The whole point is that a path check is only as good as the last path -resolution after it. Every operation here therefore runs relative to a -descriptor pinned to a directory the caller has already validated, rather than -re-resolving a name — which is what closes the swap window rather than merely -narrowing it. Where the platform has no ``*at()`` syscall family (Windows) the -identical sequence runs against full paths, the per-component ``lstat`` floor. - -**Platform bound — this guarantee is POSIX-only, deliberately.** On POSIX the -descriptor walk closes the swap window. On Windows it does not exist: there is no -``*at()`` family, so the ``lstat`` floor is all that runs, and a floor is a -check-then-use race rather than a closed window. The remedy would be -reparse-point checks (``GetFileAttributesW``, or opening with -``FILE_FLAG_OPEN_REPARSE_POINT``) and it is **not implemented, by decision rather -than by oversight**: Windows is not a supported or tested platform for this -release, and neither SDK repository has a Windows CI runner, so the checks would -ship untested — and the TypeScript SDK could not match them in any case, because -Node exposes no ``*at()`` family on *any* platform. Shipping them in Python alone -would break the cross-language parity the two SDKs are held to and would trade a -documented bound for an unverified one. - -Two consequences worth stating plainly rather than discovering later. First, on -Windows write permission on the managed root is the *only* boundary, so the -privilege-separated deployment the README documents is not advice there but the -mitigation. Second, this bound retroactively lowers the priority of the Windows -reserved-device-name work in ``skills_fs.py`` (``_WINDOWS_RESERVED_NAMES``): that -code stays, because it is cheap and it keeps a managed root written on Linux -usable when read from Windows, but it should not be read as evidence that Windows -is a hardened target. It is not. Revisit both together if Windows becomes -supported. +Nothing here knows what a skill is: this is the "write a file under a directory +an attacker may be racing you for" problem, solved once. ``skills_fs.py`` is the +only caller today. + +A path check is only as good as the last path resolution after it. Every +operation here therefore runs relative to a descriptor pinned to a directory the +caller has already validated, rather than re-resolving a name — which is what +closes the swap window rather than merely narrowing it. + +**This guarantee is POSIX-only.** On POSIX the descriptor walk closes the swap +window. Windows has no ``*at()`` syscall family, so only the per-component +``lstat`` floor runs there, and a floor is a check-then-use race rather than a +closed window; closing it would need reparse-point checks +(``GetFileAttributesW``, or opening with ``FILE_FLAG_OPEN_REPARSE_POINT``), which +this release does not implement. The practical consequence: on Windows, write +permission on the managed root is the *only* boundary, so the privilege-separated +deployment the README describes is the mitigation there rather than merely good +advice. """ from __future__ import annotations @@ -72,26 +57,20 @@ managed root can be performed relative to a descriptor pinned to a directory this module has already verified rather than re-resolved from its path. -That is what closes the swap window rather than merely narrowing it: -a descriptor refers to the inode that was checked, so replacing ``/`` -with a symlink after the check cannot redirect a write or an unlink out of the -root. POSIX has these calls; Windows does not, and there the per-component -``lstat`` floor the spec permits applies instead. - -``os.mkdir`` and ``os.rmdir`` are in the set because the managed root is now -pinned for a whole reconcile and the per-skill directory is created and removed -relative to that descriptor. Both are registered by CPython under the names -called here, so unlike the two below they need no indirection. - -The probe deliberately names ``os.rename`` and ``os.stat`` rather than the -``os.replace`` and ``os.lstat`` this module actually calls. ``os.supports_dir_fd`` -is populated per underlying syscall, and CPython registers ``renameat`` under -``rename`` only and ``fstatat`` under ``stat`` only — even though ``os.replace`` -is the same ``renameat``-backed function and ``os.lstat`` is ``fstatat`` with +A descriptor refers to the inode that was checked, so replacing +``/`` with a symlink after the check cannot redirect a write or an +unlink out of the root. POSIX has these calls; Windows does not, and there the +per-component ``lstat`` floor applies instead. + +**Do not "correct" the names in this probe.** It deliberately names +``os.rename`` and ``os.stat`` rather than the ``os.replace`` and ``os.lstat`` +this module actually calls, because ``os.supports_dir_fd`` is populated per +underlying syscall: CPython registers ``renameat`` under ``rename`` only and +``fstatat`` under ``stat`` only, even though ``os.replace`` is the same +``renameat``-backed function and ``os.lstat`` is ``fstatat`` with ``AT_SYMLINK_NOFOLLOW``, and both accept the descriptor keywords wherever their -advertised twin does (verified on CPython 3.12 and 3.13, macOS). Probing the -names this module calls would report "unsupported" on every POSIX platform and -silently disable the defense. +advertised twin does. Probing the names this module calls would report +"unsupported" on every POSIX platform and silently disable the defense. """ @@ -122,12 +101,6 @@ def open_directory_nofollow( redirects the open. Given a parent descriptor the bare name is resolved inside the inode that was checked instead, and there is nothing left to swap. - Everything the caller does afterwards goes through the returned descriptor - instead of the path, which is what turns the "narrow window" into no - window at all: the descriptor names the inode that was checked, so swapping - the path for a symlink between the check and the write cannot redirect the - write out of the managed root. - On a platform without the ``*at()`` family (Windows) this returns ``None`` after verifying via ``lstat`` that the path is a real, non-symlink directory — the per-component floor. It must not attempt the descriptor @@ -364,10 +337,9 @@ def atomic_write( Without one (Windows) the identical sequence runs against full paths, which is the per-component ``lstat`` floor. - ``os.replace`` is the one and only rename call site, reached by attribute - lookup on the ``os`` module so tests can intercept it; ``os.rename`` must - not be substituted for it (it is also the only one with defined overwrite - semantics on Windows). + ``os.replace`` is the one and only rename call site. ``os.rename`` must not + be substituted for it: only ``os.replace`` has defined overwrite semantics + on Windows. """ at_fd = dir_fd if dir_fd is not None and SUPPORTS_DIR_FD else None prefix = temp_name_prefix(name) @@ -385,8 +357,8 @@ def atomic_write( try: try: # fchmod, not chmod: operating on the descriptor cannot be redirected - # by anything that swaps the temp path underneath us, and it makes the - # mode independent of the process umask (both creation paths open 0600). + # by a swap of the temp path, and it makes the mode independent of + # the process umask (both creation paths open 0600). if _SUPPORTS_FCHMOD: os.fchmod(fd, _FILE_MODE) elif at_fd is None: @@ -432,10 +404,8 @@ def atomic_write_in(directory: Path, name: str, data: bytes) -> None: The descriptor is taken with ``O_NOFOLLOW``, so a directory swapped for a symlink between the caller's checks and the write fails it rather than redirecting it. That still leaves the directory's *ancestors* re-resolved on - the open, so this is the right primitive only where the caller holds nothing - better. ``skills_fs`` no longer does — it pins the managed root for the whole - reconcile and writes the manifest with ``atomic_write(..., dir_fd=root_fd)`` - — so prefer passing a descriptor over reaching for this. + the open, so prefer ``atomic_write`` with a descriptor the caller already + holds; this is for callers that hold nothing better. """ with pinned_directory(directory) as dir_fd: atomic_write(directory, name, data, dir_fd=dir_fd) diff --git a/packages/client/src/launchdarkly_ai_server/skills.py b/packages/client/src/launchdarkly_ai_server/skills.py index ce21a064..c29bafcf 100644 --- a/packages/client/src/launchdarkly_ai_server/skills.py +++ b/packages/client/src/launchdarkly_ai_server/skills.py @@ -2,22 +2,16 @@ Agent Skills — reference discovery and content accessors. The public retrieval surface: projecting the skill references a resolved AI -Config carries, and retrieving skill content through an injectable store seam. +Config carries, and retrieving skill content through a configurable store. -The three layers of the feature sit in three modules, and the dependencies run -one way only: +The feature is three modules, and the dependencies run one way only: -- ``skills_core.py`` — the store and telemetry seams, module state, integrity +- ``skills_core.py`` — the store interface, module state, integrity verification, and store resolution. Shared, and imports neither of the others. - ``skills.py`` (this file) — ``skill_refs``, the accessors, and ``InMemorySkillStore``. -- ``skills_fs.py`` — the highest-blast-radius layer, the one that writes to a - customer's disk. It owns the manifest format and the on-disk filenames; - nothing here knows about the filesystem. - -``_set_store``, ``_set_emitter_for_testing`` and ``_clear_state`` live here -because this module is the documented injection path; the -state they mutate lives in ``skills_core``. +- ``skills_fs.py`` — materialization onto disk. It owns the manifest format and + the on-disk filenames; nothing here knows about the filesystem. """ from __future__ import annotations @@ -50,17 +44,14 @@ # Injection points # --------------------------------------------------------------------------- # -# These three names are the documented seam: ``init_client`` and ``shutdown`` -# call them, and tests inject through them. They delegate to ``skills_core``, -# which owns the state, so that there is exactly one store and one emitter no -# matter which layer reaches for them. - - -# Bound directly to the implementations rather than wrapped: a one-line -# delegation per name would give every state mutation two definitions and two -# docstrings to keep in agreement, which is the drift these names exist to -# avoid. ``_set_emitter_for_testing`` keeps its distinct name because it has no -# production caller. +# ``init_client`` and ``shutdown`` reach the configured store through these +# names. They delegate to ``skills_core``, which owns the state, so there is +# exactly one store and one emitter no matter which layer reaches for it. +# +# Bound directly to the implementations rather than wrapped: a delegating +# one-liner per name would give every state mutation two definitions to keep in +# agreement. ``_set_emitter_for_testing`` keeps its distinct name because it has +# no production caller. _set_store = skills_core.set_store _set_emitter_for_testing = skills_core.set_emitter _clear_state = skills_core.clear_state @@ -70,10 +61,9 @@ class InMemorySkillStore: """ A skill store backed by plain dicts. - Ships for local development, tests, and bring-your-own-content injection. - Holds raw wire objects verbatim and performs no validation of its own — - verification belongs at the accessor boundary, where it applies to every - store equally. + For local development, tests, and bring-your-own-content. Holds raw wire + objects verbatim and performs no validation of its own: verification belongs + at the accessor boundary, where it applies to every store equally. Several versions of one key coexist here, because they coexist in a real delivery payload: the newest version of every skill, plus every version a @@ -114,9 +104,8 @@ def put(self, raw: dict[str, Any]) -> None: ``(key, version)`` twice replaces it. Notifies every skill-kind listener with the raw object as a single - positional argument. No validation happens here — verification belongs at - the accessor boundary, where it applies to every store equally — so a - listener sees exactly what was put, unverified. + positional argument. No validation happens here, so a listener sees + exactly what was put, unverified. """ key = raw.get("key") if not isinstance(key, str): @@ -201,9 +190,9 @@ def skill_refs(config: AiConfigRep | None) -> list[SkillReference]: per-context resolution: ``await get_skills(skill_refs(config))``. A config that came through ``parse_ai_config`` never contains an invalid - entry — parsing fails closed on one. A hand-built dict can, and a silently - shortened projection would let ``write_skills`` prune the dropped skill's - on-disk copy, so every dropped entry is logged. + entry, since parsing fails closed on one. A hand-built dict can, and a + silently shortened projection would let ``write_skills`` prune the dropped + skill's on-disk copy, so every dropped entry is logged. """ if not isinstance(config, dict): return [] @@ -270,10 +259,10 @@ async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutc """ Retrieves one verified skill, reporting *why* when there is none. - Same retrieval, same verification, same telemetry as ``get_skill`` — the two + Same retrieval, same verification, same telemetry as ``get_skill``; the two differ only in what they report. ``get_skill`` collapses "no such skill", "the store raised", "that is not the version held", and "the content failed - integrity verification" to one ``None``; this returns a ``SkillOutcome`` + integrity verification" to one ``None``. This returns a ``SkillOutcome`` whose ``reason`` names which of them happened, so a caller can fail closed on suspected tampering while tolerating a merely-absent skill: @@ -289,10 +278,10 @@ async def get_skill_result(key: str, *, version: int | None = None) -> SkillOutc failure mode, never any skill content or filesystem path. Branch on ``reason``, not on ``detail``. - Emits nothing of its own: an integrity failure has already recorded its log - record and its signal inside verification, and recording a second here would - double-count one failure. Raises ``RuntimeError`` only when no skill store is - configured, exactly as ``get_skill`` does. + Emits nothing of its own: verification has already recorded the log record + and the signal, and a second here would double-count one failure. Raises + ``RuntimeError`` only when no skill store is configured, as ``get_skill`` + does. """ resolved = resolve_from_store(require_store(), key, version) return SkillOutcome( diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 6577c597..36256207 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -1,38 +1,19 @@ """ Agent Skills — the internals ``skills`` and ``skills_fs`` both need. -Extracted so the two layers above it share one implementation through an -explicit surface instead of reaching into each other's privates. Everything -here is package-internal — nothing in this module is exported from -``launchdarkly_ai_server`` except the two constants that are public API — and -the dependency runs one way: this module imports neither ``skills`` nor -``skills_fs``. - -What lives here, and why it has to be one copy: - -- **The store seam and the configured store.** One place holds the store, so - the accessors and the materialization path cannot disagree about whether one - is configured. -- **The telemetry seam.** Every signal the feature can emit is constructed by a - ``record_*`` function in this file and nowhere else, which is what makes the - three-signal allowlist enforceable by reading one section. ``emit`` is never - called from outside this module. -- **Integrity verification.** ``verified_bytes`` runs twice per skill by design - — once at the accessor boundary, and again immediately before a write, since a - ``Skill`` can also be constructed directly by a caller. Sharing the - implementation is what keeps the two passes from drifting — the signal's - property keys must match whichever layer caught the defect. -- **Store resolution.** ``resolve_from_store`` is the fetch-and-verify sequence - the accessors and the reconcile share, so its call sites cannot drift apart — - in particular on how a raising store is handled. +Package-internal: nothing here is exported from ``launchdarkly_ai_server`` +except the two constants that are public API, and the dependency runs one way — +this module imports neither ``skills`` nor ``skills_fs``. + +It holds the store interface and the configured store, the telemetry emitter, +integrity verification, and store resolution. Each lives here in one copy so +that the accessors and the materialization path cannot disagree: about whether a +store is configured, about which signals exist, about what verification accepts, +or about how a raising store is handled. Everything the store hands back is untrusted input; the transport is not part of the trust boundary. Key, version, size, and content hash are revalidated here on every pass. - -The store and emitter are injected through ``skills._set_store`` and -``skills._set_emitter_for_testing`` — those names are the documented seam, -and they delegate here. """ from __future__ import annotations @@ -53,18 +34,11 @@ """ The kind this SDK asks a store for. -An **internal seam value**, deliberately not exported from the package root. It -is the string ``skills.py`` and ``skills_fs.py`` pass to ``SkillStore.get_object`` -and ``SkillStore.all_objects``, and a store adapter is free to map it onto -whatever the transport underneath actually uses — the value happens to match -the kind LaunchDarkly's delivery channel uses today, but a transport that spelt -it differently would translate, and that translation is the adapter's job. - -Exporting it would publish an SDK-side seam string as though it were the wire -contract, which is a claim this side cannot make and would be hard to walk back -once a caller depends on it. A store that needs to agree on a kind agrees with -whatever the SDK hands it, which is this constant reached through -``launchdarkly_ai_server.skills_core``. +Internal, and deliberately not exported from the package root: it is what the +accessors pass to ``SkillStore.get_object`` and ``SkillStore.all_objects``, and +a store adapter is free to map it onto whatever its transport uses underneath. +A store that needs to agree on a kind agrees with whatever the SDK hands it, +reached through ``launchdarkly_ai_server.skills_core``. """ MAX_SKILL_CONTENT_BYTES = 10 * 1024 * 1024 @@ -72,27 +46,23 @@ Hard cap on skill content. Legitimately delivered skills are well under this bound, so anything larger is withheld regardless of whether its hash checks out. -Set well above the platform's own limit on purpose. This is a backstop against -absurd input, not a second enforcement of the real bound: the platform refuses -oversized skills before they are ever delivered, and a client-side number sitting -just above that one would turn every backend increase into an SDK release. The -headroom lets the real limit grow without this constant moving. - -Deliberately **not** exported from the package root, unlike the on-disk and -on-the-wire constants beside it. Those are values this SDK defines and a caller -may need to agree with; this one is a local enforcement bound on content the -platform produces, so publishing it would semver-lock a number this side does -not own — and a caller pre-flighting "will my skill fit?" against it would be -reading the client's guess rather than the real limit. The reason string from -``verified_bytes`` already reports the bound when it is what withheld content. +Set well above LaunchDarkly's own limit on purpose. This is a backstop against +absurd input, not a second enforcement of the real bound, so the headroom lets +that bound grow without this constant moving. + +Not exported from the package root, unlike the on-disk and on-the-wire constants +beside it: it is a local enforcement bound rather than a value a caller needs to +agree with, and a caller pre-flighting "will my skill fit?" against it would be +reading the client's guess rather than the real limit. ``verified_bytes`` +reports the bound in its reason string when it is what withheld content. """ _LANGUAGE = "python" _SHA256_HEX = re.compile(r"\A[0-9a-f]{64}\Z") """What a legitimate content hash looks like. Anything else is redacted before -it reaches telemetry — ``contentHash`` is attacker-controlled, and a store that -put the skill body there would otherwise leak it into a signal.""" +it reaches telemetry: ``contentHash`` is untrusted, and a store that put the +skill body there would otherwise leak it into a signal.""" _SIGNAL_INTEGRITY_FAILURE = "AgentControl Skill Integrity Failure" _SIGNAL_MATERIALIZED = "AgentControl Skill Materialized" @@ -102,8 +72,8 @@ """ Stable event identity for the local integrity-failure log record. -A **compatibility surface**, not an implementation detail: the README documents -it as the string a customer's SIEM matches on, so it must never be renamed. +A compatibility surface, not an implementation detail: this is the string a SIEM +matches on, so it must never be renamed. It appears verbatim **in the message text**, not only in ``extra``. Severity alone cannot discriminate — ``list_raw_objects`` and ``resolve_from_store`` in @@ -126,13 +96,10 @@ "hash_mismatch", ] """ -The closed ``reason_code`` vocabulary — one token per ``record_integrity_failure`` -call site, and the same eight tokens in every language implementation of this -feature, so a detection rule written against one SDK reads the others. - -A ``Literal`` rather than a bare ``str`` so a typo at a call site is a type -error, and so widening the vocabulary is a deliberate edit here rather than a -new string invented at the site that needed it. +The closed ``reason_code`` vocabulary — one token per +``record_integrity_failure`` call site. Stable: a detection rule written against +these tokens keeps working, so adding one is a deliberate edit here rather than +a new string invented at the call site that needed it. """ INTEGRITY_REASON_CODES: frozenset[str] = frozenset(get_args(IntegrityReasonCode)) @@ -145,17 +112,14 @@ "development and testing." ) """ -The first thing a user sees when no store is configured, so it names both stores. - -``FDv2SkillStore`` comes first because it is the answer in production, and a -message that offered only ``InMemorySkillStore`` would point a deployment at the -development store. Callers match on "skill store"; keep that phrase if the -wording changes. +The first thing a user sees when no store is configured, so it names both stores, +``FDv2SkillStore`` first because it is the answer in production. Callers match on +"skill store"; keep that phrase if the wording changes. """ # --------------------------------------------------------------------------- -# The store seam +# The store interface # --------------------------------------------------------------------------- @@ -166,28 +130,26 @@ class SkillStore(Protocol): Duck-typed on purpose, mirroring how the LaunchDarkly client interface works in this package: pass any object carrying these methods. - ``is_initialized()``, ``add_listener(kind, fn)`` and - ``remove_listener(kind, fn)`` are part of the interface but **optional**, - which is why they are deliberately not declared here: a Protocol member is required for structural compatibility, so declaring - them would reject every store that does not implement them. Nothing in this - module calls either — they exist for the delivery transport to push updates - through, and for a consumer such as ``watch_skills`` to stop receiving them. - A store that implements ``add_listener`` should implement ``remove_listener`` - too; consumers probe for it and skip detaching when it is absent, so an - older store keeps working at the cost of a listener that lives as long as - the store does. ``remove_listener`` removes one occurrence of *fn* under - *kind* and is a no-op when *fn* is not registered. + Three members are **optional**, and are deliberately not declared here: a + Protocol member is required for structural compatibility, so declaring them + would reject every store that does not implement them. Each is probed for + instead, and each has a defined behaviour when absent. ``is_initialized()`` reports whether the store has received its initial data - — for a delivery transport, whether a payload has arrived yet. A store that - does not implement it is treated as initialized, which is correct for one - that is populated by hand. It matters because "the store holds nothing" and - "the store has not heard yet" are the same answer through ``all_objects``, - and ``write_skills("*")`` reads the first as "every skill was revoked": see - ``store_is_initialized``. + — for a delivery transport, whether a payload has arrived yet. Absent means + initialized, which is right for a store populated by hand. It matters + because "the store holds nothing" and "the store has not heard yet" are the + same answer through ``all_objects``, and ``write_skills("*")`` would read the + first as "every skill was revoked": see ``store_is_initialized``. + + ``add_listener(kind, fn)`` lets a delivery transport push updates, and + ``remove_listener(kind, fn)`` lets a consumer such as ``watch_skills`` stop + receiving them; it removes one occurrence of *fn* under *kind* and is a + no-op when *fn* is not registered. A store offering the first should offer + the second: consumers skip detaching when it is absent, so such a store + works at the cost of a listener that lives as long as it does. - The raw objects a store serves are wire-shaped, with camelCase field names - identical across language implementations:: + The raw objects a store serves are wire-shaped, with camelCase field names:: {"key": "pdf-extraction", "version": 2, "content": "---\\n...", "contentHash": "9f3a...", "name": "PDF Extraction", "description": "..."} @@ -195,11 +157,10 @@ class SkillStore(Protocol): **Version is part of the lookup identity, not a filter applied afterwards.** A delivery payload holds the newest version of every skill *and* every version any variation currently pins, so two versions of one key coexist - routinely. A seam keyed by key alone cannot express "the one this variation - pinned": it would answer with the newest and the caller would then have to - reject it, which turns a pinned reference into a missing skill. So - ``get_object`` takes the wanted version, and ``version=None`` means "the - newest you hold". + routinely. An interface keyed by key alone cannot express "the one this + variation pinned": it would answer with the newest, and the caller rejecting + it turns a pinned reference into a missing skill. So ``get_object`` takes the + wanted version, and ``version=None`` means "the newest you hold". ``all_objects`` returns one entry per *(key, version)* the store holds. Its dict keys are **opaque store-internal identifiers** — do not parse them, and @@ -216,7 +177,7 @@ def all_objects(self, kind: str) -> dict[str, dict[str, Any]]: ... # --------------------------------------------------------------------------- -# Telemetry seam +# Telemetry # --------------------------------------------------------------------------- @@ -228,12 +189,11 @@ class _NoOpEmitter: """ The default emitter. - No skills telemetry leaves the process in this release: ``client.track()`` is - the wrong channel (it needs an LD context, spends the customer's event - volume, and lands in their data export), and the diagnostic-event channel - that would be right has no wrapper-SDK extension point yet. Signals are - recorded through this seam so the eventual transport drops in behind it - without touching a single call site. + No skills telemetry leaves the process: ``client.track()`` is the wrong + channel for it — that needs an LD context, spends the application's event + volume, and lands in its data export. Signals are still constructed and + recorded, so a transport can be installed behind this interface without + touching a call site. """ def record(self, signal: str, properties: dict[str, Any]) -> None: @@ -254,24 +214,21 @@ def record(self, signal: str, properties: dict[str, Any]) -> None: def set_store(store: Any) -> None: """ - Replaces the configured store. - - Reached through ``skills._set_store``, which is the documented seam; see that - function for who calls it and why it has no test-only twin. + Replaces the configured store. Reached through ``skills._set_store``, the + documented injection point. """ global _store _store = store def set_emitter(emitter: Any) -> None: - """Replaces the telemetry emitter. Reached through - ``skills._set_emitter_for_testing``.""" + """Replaces the telemetry emitter.""" global _emitter _emitter = emitter def clear_state() -> None: - """Drops both the store and the emitter. Reached through ``skills._clear_state``.""" + """Drops both the store and the emitter.""" global _store, _emitter _store = None _emitter = _NOOP_EMITTER @@ -302,7 +259,7 @@ def store_is_initialized(store: Any) -> bool: yet received a payload as an environment whose every skill was revoked. Retrieval through such a store is reported unavailable, which suppresses pruning — the same treatment a raising store gets, and for the same reason: - deleting a customer's files because content could not be retrieved would + deleting the application's files because content could not be retrieved would turn a slow boot into data loss. A probe that raises counts as not initialized. A store that cannot answer @@ -349,20 +306,16 @@ def record_integrity_failure( Carries hashes and byte counts only — the skill body never appears in a signal, a log line, or an error message. - The two surfaces are deliberately different sizes, and the log record is the - more important of the two. The signal is product telemetry: opt-out - respecting, no-op by default, and its property set is a documented allowlist - (``agents.md``) that does not grow. The **log record** is the customer-owned - detection path — the only one that works when telemetry is off, and the only - one that exists at all in an instance with no telemetry destination — so it - is designed to be ingested and alerted on, and additionally carries the - stable event name, the action taken, the human-readable reason, and the - machine-parseable ``reason_code``. - - Emitted twice over, because neither form alone is sufficient: the message - text carries ``INTEGRITY_FAILURE_EVENT`` followed by compact JSON, so the - record survives ``logging.basicConfig()`` and is greppable and ``jq``-able - under any handler configuration; ``extra["ld_skills"]`` carries the same + The two are deliberately different sizes. The signal is product telemetry: + no-op by default, with a fixed property set. The log record is the + application's own detection path — the only one that works when telemetry is + off — so it additionally carries the stable event name, the action taken, the + human-readable reason, and the machine-parseable ``reason_code``. + + The record is emitted in two forms because neither alone is enough: the + message text carries ``INTEGRITY_FAILURE_EVENT`` followed by compact JSON, so + it survives ``logging.basicConfig()`` and is greppable and ``jq``-able under + any handler configuration, and ``extra["ld_skills"]`` carries the same mapping unflattened for a structured handler that would rather not reparse. """ # Both of these come off the wire, so neither may be echoed verbatim: a store @@ -394,11 +347,10 @@ def record_integrity_failure( "reason": reason, **properties, } - # ``sort_keys`` is load-bearing rather than cosmetic: the other language - # implementations build this object in alphabetical key order, so sorting - # here makes the serialized line byte-identical across SDKs for the same - # input, modulo ``language``. Do not drop it, and do not reorder the keys - # above expecting the output to follow. + # ``sort_keys`` is part of the record's format rather than cosmetic: it is + # what makes the serialized line stable for a given input, so a detection + # rule can match on it. Do not drop it, and do not reorder the keys above + # expecting the output to follow. logger.error( "%s %s", INTEGRITY_FAILURE_EVENT, @@ -414,7 +366,7 @@ def record_materialized( """ Records a materialization. Deliberately carries no ``target_path`` and no filesystem path of any kind — the same reasoning that keeps the skill body - out of telemetry keeps the customer's directory layout out. Paths live in the + out of telemetry keeps the application's directory layout out. Paths live in the returned ``ReconcileReport``, which is user-facing API rather than telemetry. """ emit( @@ -433,10 +385,9 @@ def record_revoked(skill_key: str, version: Any) -> None: """ Records a revocation — a prune that removed a formerly managed skill. - Lives here with the other two recorders rather than at the prune site so the - signal allowlist is maintained in one place: every signal this SDK can emit - is visible in this section of this module, and nothing outside it touches - ``emit``. + Lives here with the other two recorders rather than at the prune site, so + every signal this SDK can emit is visible in one place and nothing outside + this module touches ``emit``. """ # Both fields come off the manifest, which is untrusted — same rule as # ``record_integrity_failure``: shape-check, then redact, so a hand-edited @@ -488,25 +439,16 @@ def verified_bytes( Returns the verbatim bytes and their locally computed sha256, or a human-readable reason — having already recorded the integrity signal, so the - signal's property set cannot depend on which caller noticed. The hash handed - back is the one computed here, never the caller's expected value: the two are - equal on this path by construction, and returning the locally derived one - keeps an attacker-supplied string out of ``Skill``. - - The two outcomes are distinct types rather than a ``tuple | str`` union so a - call site reads as "verification failed" instead of "the result is a string", - and so a future success payload carrying a ``str`` cannot silently invert the - discrimination. + signal does not depend on which caller noticed. The hash handed back is + always the one computed here, never the caller's expected value, which keeps + an untrusted string out of ``Skill``. This runs twice per skill by design: once at the accessor boundary, and again immediately before a write, because a ``Skill`` can also be constructed - directly by a caller. Sharing the implementation is what keeps those two - passes from drifting — the property keys must match. - - The second pass re-hashes bytes the first pass already hashed. That - redundancy is deliberate: it is negligible next to the write it guards, and - carrying the first pass's verdict forward would put a "trust the value - computed upstream" branch inside the one function whose entire job is not to. + directly by a caller. The second pass re-hashes bytes the first pass already + hashed, which is negligible next to the write it guards — and carrying the + first pass's verdict forward would put a "trust the value computed upstream" + branch inside the one function whose job is not to. """ if isinstance(content, bytes): encoded = content @@ -633,14 +575,12 @@ def log_withholding_summary(subject: str, requested: int, resolved: int) -> None One WARN per run when content was withheld, naming the counts. Every individual withholding already records an integrity signal and an error - log line, but a caller reading logs at WARN sees neither. That matters most in - the case where *nothing* verified — a payload built before ``contentHash`` is - populated, say — because the feature then returns an empty result that is - indistinguishable from "this project has no skills". A run-level summary is - the difference between a silent no-op and a visible one. + line, but a caller reading logs at WARN sees neither — and the case that + matters most is a run where *nothing* verified, because the result is then an + empty list indistinguishable from "this project has no skills". - Called once per batch retrieval, not once per skill, so a large withholding - run does not itself become the noise. + Called once per batch, not once per skill, so a large withholding run does + not itself become the noise. """ withheld = requested - resolved if withheld <= 0: @@ -679,10 +619,8 @@ def list_raw_objects( result themselves; ``newest_by_key`` does it. Returns the reason rather than raising, because both callers need the - distinction between "no skills" and "the store is broken" — and they need it - worded identically. Letting the exception out instead would make each of - them re-derive the log line and the message, which is the drift this module - exists to prevent. + distinction between "no skills" and "the store is broken", worded + identically. An answer that is not a mapping is a broken store, on the same footing as one that raised — **not** an empty one. Collapsing it to ``{}`` would make a @@ -711,22 +649,19 @@ def newest_by_key(objects: dict[str, dict[str, Any]]) -> list[tuple[str, Any]]: One raw object per skill key — the highest version of each, paired with the store key it was served under. - ``all_objects`` may hold several versions of one key, and both callers that - consume the whole store want one skill per key: ``all_skills`` because a list - holding two versions of one key is not a set of skills, and the ``"*"`` - reconcile because ``//SKILL.md`` is a single path and writing it - twice in one run is a bug rather than a policy. - - The store key is carried through rather than discarded because the reconcile - attributes a failure to it when the object's own key is unusable. + ``all_objects`` may hold several versions of one key, and both whole-store + callers want one skill per key: ``all_skills``, because a list holding two + versions of one key is not a set of skills, and the ``"*"`` reconcile, + because ``//SKILL.md`` is a single path. The store key is carried + through because the reconcile attributes a failure to it when the object's + own key is unusable. An object too malformed to carry a usable key and version is **kept**, so - verification is what withholds it: a silently dropped object falls out of - the requested set, and prune would then delete the last known-good copy - already on disk. The exception is an object whose skill key resolved anyway - from another version — there the resolved object already holds the key in - the requested set, so keeping the malformed one would only report a - withholding for a key that in fact resolved. + verification is what withholds it: dropped silently it would fall out of the + requested set, and prune would then delete the last known-good copy on disk. + The exception is an object whose key resolved from another version anyway — + that key is already in the requested set, so keeping the malformed sibling + would only report a withholding for a key that resolved. """ best: dict[str, tuple[str, Any]] = {} unusable: list[tuple[str, Any]] = [] @@ -765,20 +700,15 @@ class Resolution: Which of the five public outcomes this resolution is. Declared first and **without a default**, so every construction site has to - state it. A default would be the wrong shape twice over: a contributor - adding a sixth internal outcome would inherit whichever token happened to be - the default rather than deciding which public token it maps to, and if that - default were ``"ok"`` a failure would publish ``ok`` with no skill attached. + state which public token it maps to rather than inheriting one. - Carried as a token rather than derived from ``error`` on the way out: - ``get_skill_result`` publishes this value, and pattern-matching prose to - recover a decision a caller fails closed on is exactly the fragility the - typed outcome exists to remove. A reviewer can read the mapping here. + ``get_skill_result`` publishes this value directly. It is carried as a token + rather than recovered from ``error``, because pattern-matching prose for a + decision a caller fails closed on is the fragility the typed outcome removes. - Distinct from ``unavailable`` on purpose — that flag answers one question - (may prune run?) and this token answers a different one (what does the - caller learn?) — but the two can only disagree by a bug: ``unavailable`` is - ``True`` in exactly the ``store_unavailable`` case. + Distinct from ``unavailable``, which answers a different question (may prune + run?), but the two can only disagree by a bug: ``unavailable`` is ``True`` in + exactly the ``store_unavailable`` case. """ skill: Skill | None = None error: str | None = None @@ -795,10 +725,8 @@ def resolve_from_store( ) -> Resolution: """ Fetches one key and verifies it — the sequence the accessors and the - materialization path share. - - Written once on purpose, so the call sites cannot drift apart — in - particular on the policy for a raising store. + materialization path share, written once so the two cannot drift apart on + the policy for a raising store. ``wanted_version`` goes *into* the lookup, because a store may hold several versions of one key and only it can pick between them; ``None`` asks for the diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 702ca30f..e0934470 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -24,17 +24,16 @@ - **It does not verify content.** Verification lives at the accessor boundary in ``skills_core`` so that it applies to every store equally, including a - customer's own. + a store the application supplies itself. - **It does not work around a missing ``contentHash``.** A hashless object is held verbatim and *withheld* by verification with ``missing_content_hash``. This module's job is to make that outcome loud — see ``StoreDiagnostics``. - **It does not evaluate anything.** Flag and segment objects that share the connection are skipped and counted, nothing more. -The design rationale — why the skill's version is read from the object's -``key`` and never from ``version``, why changes commit at -``payload-transferred``, why there is one network timeout — is in -``agents.md`` under *The delivery transport*. +Why the skill's version is read from the object's ``key`` and never from +``version``, why changes commit at ``payload-transferred``, and why there is one +network timeout are each explained where the code does it. """ from __future__ import annotations @@ -70,9 +69,9 @@ Object kinds on the SDK-facing channel are open strings: the agent-skill payload is classified ``generic`` and every object in it carries the kind its producer registered, which for skills is the bare category name. Delivery lower-cases the -kind, so an exact comparison is the whole test. The kind happens to equal -``skills_core.SKILL_OBJECT_KIND`` today; they are still separate constants, -because one is a wire value LaunchDarkly owns and the other is an SDK seam. +kind, so an exact comparison is the whole test. The value happens to equal +``skills_core.SKILL_OBJECT_KIND``; they remain separate constants, because one is +a wire value LaunchDarkly owns and the other is what this SDK asks a store for. """ FDV2_KEY_DELIMITER = ":" @@ -493,35 +492,34 @@ class _TransferOutcome: disconnect: str | None = None up_to_date: bool = False """ - The server said what we hold is current and it has nothing to transfer. + The server said the content held is current and it has nothing to transfer. - A complete answer that commits nothing, which is exactly what a 304 is to a - poll. The delivery loop counts it as a healthy connection; see + A complete answer that commits nothing, which is what a 304 is to a poll. + The delivery loop counts it as a healthy connection; see ``FDv2SkillStore._apply``. """ class _ProtocolReader: """ - Applies FDv2 events to an object set. Pure — no sockets, no threads, no clock — - so every wire case is testable without a server. + Applies FDv2 events to an object set. Pure — no sockets, no threads, no + clock — so every wire case is testable without a server. **Changes are buffered and committed at ``payload-transferred``.** A payload version is the unit of consistency: applying half of one would publish a state the server never described, and on a full transfer would briefly empty the store. Listeners therefore fire once per commit, not once per object. - **The first payload intent is read, and is assumed to be the skill payload.** + **The first payload intent is read, and is taken to be the skill payload.** Delivery provides one payload per credential and the protocol requires a - client to ignore all but the first payload intent, so ``payloads[0]`` is both - what arrives and what the protocol says to read. If that ever widens, an - ``xfer-full`` for somebody else's payload would empty the skill set and the - next ``payload-transferred`` would publish it empty — with pruning on, the - difference between a reconcile and deleting a customer's files. This layer - therefore learns which payload skills arrive on and declines to apply a - transfer of any other, once at WARNING and counted. The residual is the first - transfer of a connection: before a skill has arrived there is nothing to - compare a payload against. + client to ignore all but the first intent, so ``payloads[0]`` is both what + arrives and what the protocol says to read. Should that ever widen, an + ``xfer-full`` for another payload would empty the skill set and the next + ``payload-transferred`` would publish it empty — with pruning on, the + difference between a reconcile and deleting the application's files. So this + layer learns which payload skills arrive on and declines to apply a transfer + of any other, warning once and counting it. The residual case is the first + transfer of a connection, where there is nothing to compare against yet. """ def __init__(self, committed: _SkillObjectSet) -> None: @@ -638,7 +636,7 @@ def _delete_object(self, data: Any) -> _TransferOutcome: return _TransferOutcome() target.delete(tombstone) self.diagnostics.objects_revoked += 1 - # A revocation identifies the payload as ours just as a put does. + # A revocation identifies the skill payload just as a put does. self._skills_in_payload += 1 # A tombstone carries identity and no content; see # ``FDv2SkillStore.add_listener`` for what listeners should expect. @@ -713,9 +711,8 @@ def _is_foreign_payload(self, payload_id: str | None) -> bool: """ Whether a transfer completes a payload other than the one skills arrive on. - ``False`` unless both payloads are known, so one-payload delivery and the - first transfer of a connection behave exactly as they did before this - check existed. + ``False`` unless both payloads are known, so one-payload delivery and + the first transfer of a connection are unaffected. """ return ( self._skill_payload_id is not None @@ -729,10 +726,10 @@ def _warn_multiple_payloads(self, payloads: list[Any]) -> None: """ One WARNING per reader for an intent describing more than one payload. - Not an error: reading only the first is what the protocol asks for. But it - means the first payload is no longer *guaranteed* to be the skill payload, - and an intent for another payload arriving before any skill has been seen - is the one case ``_is_foreign_payload`` cannot catch. + Not an error: reading only the first is what the protocol asks for. But + it means the first payload is no longer *guaranteed* to be the skill + payload, and an intent for another payload arriving before any skill has + been seen is the one case ``_is_foreign_payload`` cannot catch. """ if self._warned_multiple_payloads: return @@ -842,7 +839,7 @@ def _retry_after_seconds(headers: Any) -> float | None: ``Retry-After`` in seconds, when the server sent a usable one. The HTTP-date form, and non-finite values such as ``inf`` or ``1e309`` that - ``float`` accepts, fall back to our own backoff: none of them is a delay, + ``float`` accepts, fall back to this module's backoff: none is a delay, and an infinite one would overflow the wait that honours it. """ if headers is None: @@ -956,7 +953,7 @@ def __init__( self._sdk_key = sdk_key self._base_uri = base_uri.rstrip("/") self._read_timeout = read_timeout - # Injectable so tests can drive a fake endpoint without a socket. + # Injectable, so an alternative transport can be supplied. self._opener = opener or urllib.request.build_opener() self._lock = threading.Lock() # The response of a poll in flight, so ``interrupt`` can reach its @@ -1168,7 +1165,8 @@ class FDv2SkillStore: store = FDv2SkillStore(sdk_key=os.environ["LD_SDK_KEY"]) store.start() - store.wait_for_skills(timeout=10) + if not store.wait_for_skills(timeout=10): + ... # no payload yet: see is_initialized await init_client(options={"skillStore": store}) skill = await get_skill("pdf-extraction") @@ -1184,7 +1182,9 @@ class FDv2SkillStore: the connection and fills memory, and ``get_object`` only ever reads what has already arrived. A process that calls ``get_skill`` immediately after ``start()`` may see an empty store; ``wait_for_skills`` orders boot against - the first payload. + the first payload, and ``is_initialized`` reports the same fact without + waiting — which is what stops ``write_skills("*")`` from pruning against a + store that has not heard yet. **Last known good survives an outage.** A transport failure never empties the store and never makes ``get_object`` raise, which is what makes @@ -1585,8 +1585,8 @@ def _apply(self, name: str, data: Any) -> None: self._basis = outcome.basis if outcome.committed or outcome.up_to_date: # Both break the row of consecutive failures: a commit is a payload - # delivered, and ``up_to_date`` is the server confirming we already - # hold it. Counting only the commit would give up on a healthy + # delivered, and ``up_to_date`` is the server confirming the store + # already holds it. Counting only the commit would give up on a healthy # stream serving an environment whose skills are not changing: # nothing to transfer means no commit, while every recycled # connection still ends in a drop. diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 4d1a74d0..99fd8f48 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -1,23 +1,22 @@ """ Agent Skills — filesystem materialization. -The highest-blast-radius layer of the feature: this is the part that writes to a -customer's disk. Split out of ``skills.py`` on that boundary — everything here -takes already-verified content and reconciles it against a managed root, while -``skills.py`` owns retrieval and verification and knows nothing about the -filesystem. The dependency runs one way only, and the descriptor-pinned -primitives every destructive step goes through live in ``safe_fs.py``. +This is the part that writes to disk. Everything here takes already-verified +content and reconciles it against a managed root, while ``skills.py`` owns +retrieval and verification and knows nothing about the filesystem. The +dependency runs one way only, and the descriptor-pinned primitives every +destructive step goes through live in ``safe_fs.py``. The managed root is pinned to a descriptor once per reconcile and every destructive operation runs relative to it, so no ancestor of the root is ever -re-resolved from its path mid-run. The path checks are still there and still -run, but they are defense in depth rather than the boundary: a check on a path -is only as good as the last resolution of that path after it, and the descriptor -is what makes there be no later resolution. +re-resolved from its path mid-run. The path checks still run, but they are +defense in depth rather than the boundary: a check on a path is only as good as +the last resolution of that path after it, and the descriptor is what makes +there be no later resolution. -The reconcile is manifest-driven and fails closed: destructive operations only +The reconcile is manifest-driven and fails closed. Destructive operations only ever touch paths ``/.launchdarkly-skills.json`` records under a matching -key, a corrupt manifest suppresses every destructive action, and an incomplete +key; a corrupt manifest suppresses every destructive action; and an incomplete retrieval suppresses pruning. Content is re-verified immediately before the write, because a ``Skill`` can also be constructed directly by a caller. """ @@ -124,11 +123,9 @@ ) """ The 22 MS-DOS device names Windows still reserves, which cannot be directory -names there. The key grammar admits every one of them, so a customer who names a -skill ``con`` gets a working reconcile on Linux and a broken one on Windows — -rejected here instead, on every platform, so the on-disk result never depends on -which OS ran the write. Neither repository has a Windows CI runner, which is the -condition that produced the gap in the first place. +names there. The key grammar admits every one of them, so a skill named ``con`` +would reconcile on Linux and fail on Windows. Rejected here instead, on every +platform, so the on-disk result never depends on which OS ran the write. The bare names are the whole set: no suffix stripping is needed because the key grammar admits no ``.``, so ``con.txt`` is unreachable, and ``CONIN$`` / @@ -187,23 +184,22 @@ async def write_skills( so a root replaced after validation is refused rather than followed. The manifest is read through it too, so the record that authorizes those removals comes from inside the pinned root rather than from wherever the - root's path led by the time it was read. This guarantee is POSIX-only, for - the reasons ``safe_fs`` states at length. + root's path led by the time it was read. This guarantee is POSIX-only; see + ``safe_fs``. **This call performs synchronous filesystem I/O and does not yield.** It is - ``async`` for signature parity with the other accessors and with the - TypeScript SDK, not because it awaits anything: every read, write, ``fsync`` - and rename runs inline, so a large reconcile blocks the event loop for its - duration. Wrap it in ``asyncio.to_thread`` if that matters on your loop. - ``timeout`` is checked between steps rather than interrupting one in - progress, for the same reason. - - **One root, one reconcile at a time.** Because nothing here yields, a whole - reconcile is atomic against every other task on the loop today. Wrapping it - to run concurrently makes that the caller's problem instead: two runs - against the same root interleave on the manifest, and the loser's entries - are lost — which leaves the files it wrote unmanaged, and a later reconcile - then refuses them as files the SDK did not write. + ``async`` for signature parity with the other accessors, not because it + awaits anything: every read, write, ``fsync`` and rename runs inline, so a + large reconcile blocks the event loop for its duration. Wrap it in + ``asyncio.to_thread`` if that matters on your loop. ``timeout`` is checked + between steps rather than interrupting one in progress, for the same + reason. + + **One root, one reconcile at a time.** Because nothing here yields, a + reconcile is atomic against every other task on the loop. Wrapping it to run + concurrently makes that the caller's problem: two runs against the same root + interleave on the manifest, the loser's entries are lost, and a later + reconcile then refuses the files it wrote as files the SDK did not write. """ # Both of these are annotated as closed sets, but the values can still arrive # from untyped code, so they are checked rather than assumed. @@ -261,9 +257,9 @@ async def write_skills( incomplete = incomplete or write_timed_out # Pruning is destructive, so it needs a trustworthy picture of both - # sides: a corrupt manifest means we do not know what we own, and an + # sides: a corrupt manifest leaves the SDK unsure what it owns, and an # incomplete run — a retrieval that failed, or a deadline that expired - # mid-write — means we do not know what is still current. Either way, + # mid-write — leaves it unsure what is still current. Either way, # deleting would be a guess. if prune and manifest_error is None and not incomplete: actions.extend( @@ -371,10 +367,8 @@ def _rewrite_manifest( """ Writes the updated manifest. Returns an error action, or nothing. - Relative to the root descriptor the caller has held all along, rather than - re-opening the root by path: this used to be the one operation that pinned - the root, and it pinned it too late to matter, since every skill file was - already written by the time it ran. + Written relative to the root descriptor the caller holds, rather than + re-opening the root by path — the same reason every other step here is. """ manifest["manifestVersion"] = MANIFEST_VERSION manifest["entries"] = entries @@ -412,9 +406,9 @@ def _available_store(deadline: float, subject: str) -> SkillStore | _RetrievalBl The configured store, or why retrieval must not be attempted. Written once because this gate is what sets ``unavailable`` and therefore - suppresses pruning. If it were maintained in two places, a condition added - to one and not the other would not merely produce a wrong message — it - would delete the user's files. + suppresses pruning. Maintained in two places, a condition added to one and + not the other would not merely produce a wrong message — it would delete + the application's files. """ if time.monotonic() >= deadline: return _RetrievalBlocked( @@ -664,31 +658,24 @@ def _load_manifest( Loads the manifest. Returns ``(manifest, error)``. A manifest that cannot be read, cannot be parsed, is not an object, carries a - ``manifestVersion`` this release does not understand, or has a malformed - ``entries`` map is **corrupt**. The caller then performs no destructive - action and leaves the file itself alone: rewriting it would destroy the only - record of what the SDK owns, and acting on a manifest we cannot read would - mean guessing at which of the customer's files are ours. + ``manifestVersion`` this release does not understand, is larger than the + read cap, or has a malformed ``entries`` map is **corrupt**. The caller then + performs no destructive action and leaves the file itself alone: rewriting it + would destroy the only record of what the SDK owns, and acting on a manifest + it cannot read would mean guessing at which files those are. An absent manifest is not corrupt — that is simply a fresh root. Read relative to the root descriptor the caller holds, for the same reason - every destructive step is: this file is what decides which of the - customer's files the SDK may overwrite and delete. Read by path it - re-resolved the root on the way in, so a root swapped after the pin handed - the run somebody else's entries — and every consequence then landed in the - *real* root through the descriptor. An empty manifest from the wrong - directory made every managed file look unowned; a populated one aimed the - prune; and either way ``_rewrite_manifest`` committed the result back over - the real manifest, destroying the ownership record that protects the - customer's files on the next run. Reading it through the descriptor is what - makes the pin cover the decision as well as the actions. - - The single descriptor-relative open also replaces an ``exists()``-then-read - pair on the same path, so absence is now ``ENOENT`` on the read itself. It - is the same open the skill files get, which additionally means a symlink or - a FIFO wearing the manifest's name is refused as corruption rather than - followed or waited on. + every destructive step is: this file decides which of the application's + files the SDK may overwrite and delete, so the pin has to cover the decision + as well as the actions. Read by path, a root swapped after the pin would + hand the run somebody else's entries while every consequence landed in the + real root. + + That single open also means absence is ``ENOENT`` on the read itself, and + that a symlink or a FIFO wearing the manifest's name is refused as + corruption rather than followed or waited on. """ fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} @@ -770,8 +757,8 @@ def _unsafe_path_reason( The path defenses, in one place. Returns why ``//SKILL.md`` must not be touched, or ``None``. - Shared by the write and prune paths: ``agents.md`` marks these checks - non-relaxable, and maintaining them twice is how they drift. + Shared by the write and prune paths, and not to be relaxed in either: + maintaining these twice is how they drift. These are defense in depth and not the boundary. Every one of them inspects a path, so each is a check-then-use against anything that can rename a @@ -805,11 +792,10 @@ def _key_rejection_reason(key: Any) -> str | None: """ Why *key* must not become a directory name under the managed root, or ``None``. - Re-validated locally whatever any upstream layer already did, and - before any filesystem call, because a key becomes a path component. Shared by - the write and the prune paths so the two cannot disagree about which keys - this SDK could own; ``agents.md`` marks these checks non-relaxable, and - maintaining them twice is how they drift. + Re-validated locally whatever any upstream layer already did, and before + any filesystem call, because a key becomes a path component. Shared by the + write and prune paths so the two cannot disagree about which keys this SDK + could own, and not to be relaxed in either. ``key.encode`` is safe here only because it runs *after* the pattern check: the key grammar admits no surrogate, so there is no unencodable key left to @@ -826,11 +812,11 @@ def _key_rejection_reason(key: Any) -> str | None: f"skill key '{key[:32]}...' is {key_bytes} bytes, over the " f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" ) - # Same reasoning as the byte bound above, and it lives at the same layer for - # the same reason: the grammar itself must keep admitting these, because - # rejecting them there would fail the whole AI Config over one skill, and - # would shrink ``skill_refs`` — which is what authorizes a prune, so a - # Windows-only constraint would delete the skill's file on Linux. + # Checked here rather than in the grammar, for the same reason as the byte + # bound above: rejecting these in the grammar would fail a whole AI Config + # over one skill, and would shrink ``skill_refs`` — which is what authorizes + # a prune, so a Windows-only constraint would delete the skill's file on + # Linux. if key in _WINDOWS_RESERVED_NAMES: return ( f"skill key '{key}' is a name Windows reserves for a device and " @@ -882,20 +868,20 @@ def failed(message: str) -> ReconcileAction: # to create can never be a candidate. _sweep_orphan_temp_files(root, root_fd, key) - # Overwrite only what the manifest records as ours under this key. + # Overwrite only what the manifest records as the SDK's under this key. entry = entries.get(relative) managed = isinstance(entry, dict) and entry.get("key") == key exists = target.exists() if exists: - # Hash first, and decide from the bytes. The manifest check below is what - # protects a customer's own file, but it also refuses the file this SDK - # itself wrote and was killed before recording — the reconcile writes - # every skill and only then rewrites the manifest, so a crash in that - # window leaves a managed path with no entry, and every later reconcile - # takes the refusal branch forever. Comparing the bytes distinguishes the - # two cases without weakening anything: only content byte-identical to - # what LaunchDarkly resolved is ever adopted. + # Hash first, and decide from the bytes. The manifest check below is + # what protects a file the SDK did not write, but on its own it also + # refuses one the SDK wrote and was killed before recording: the + # reconcile writes every skill and only then rewrites the manifest, so a + # crash in that window leaves a managed path with no entry, and every + # later reconcile would take the refusal branch forever. Comparing the + # bytes separates the two cases without weakening anything, since only + # content byte-identical to what LaunchDarkly resolved is adopted. try: on_disk = _read_regular_file(target, max_bytes=len(encoded)) except OSError as exc: @@ -911,12 +897,12 @@ def failed(message: str) -> ReconcileAction: return failed(f"'{relative}' could not be read: {exc}") if hashlib.sha256(on_disk).hexdigest() == content_hash: - # ``skipped_current`` covers this deliberately, rather than a new - # action kind: its documented meaning is that the bytes on disk - # already are the resolved content, which is exactly as true for an - # adopted file as for one this SDK wrote and recorded. Adoption does - # add a manifest entry, so the file becomes prunable later — correct, - # because a prune then removes content LaunchDarkly delivered anyway. + # ``skipped_current`` rather than a new action kind: its meaning is + # that the bytes on disk already are the resolved content, which is + # as true for an adopted file as for one the SDK wrote and recorded. + # Adoption adds a manifest entry, so the file becomes prunable + # later — correct, since a prune then removes content LaunchDarkly + # delivered anyway. _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, "skipped_current") return ReconcileAction( @@ -955,31 +941,20 @@ def _read_regular_file( """ Reads *target*, refusing anything that is not a regular file. - A plain ``Path.read_bytes`` would ``open()`` by name — and opening a FIFO - with no writer blocks forever, so an attacker who can swap the managed file - for one (the same capability the symlink checks defend against) could hang - the whole reconcile, and the event loop with it. ``O_NONBLOCK`` makes that - open return immediately (it is a no-op for regular files), ``O_NOFOLLOW`` - refuses a trailing symlink, and the ``fstat`` on the descriptor — not the - path — is what the type check trusts. ``O_BINARY`` is what keeps these - bytes the *verbatim* bytes: it is 0 on POSIX, but on Windows a descriptor - without it translates CRLF on read, which would fail the hash comparison - against content that is actually current. - - With *dir_fd*, *target* is a bare component resolved against that - descriptor rather than a path the kernel walks from the root down. The - manifest read passes the root descriptor, because the manifest is what - decides which of the customer's files the SDK may touch. ``_write_one``'s - comparison read still resolves a path: what it names is - ``/SKILL.md``, so covering it the same way needs a descriptor for the - skill directory rather than for the root. - - ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes, so no read here - can be made to pull an arbitrary file into memory. The extra byte is what - lets a caller tell "at the cap" from "over it" without reading the rest: - the comparison consumer hashes the result and anything longer than the - resolved content cannot match it, and the manifest reader reports the - overage as corruption. + Each flag earns its place. ``O_NONBLOCK`` because opening a FIFO with no + writer blocks forever, so a managed file swapped for one would hang the + reconcile and the event loop with it (it is a no-op for regular files). + ``O_NOFOLLOW`` to refuse a trailing symlink. ``O_BINARY`` to keep the bytes + verbatim: 0 on POSIX, but without it a Windows descriptor translates CRLF on + read, which would fail the hash comparison against current content. The type + check reads ``fstat`` on the descriptor, never the path. + + With *dir_fd*, *target* is a bare component resolved against that descriptor + rather than a path the kernel walks from the root down. + + ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes, so no read here can + be made to pull an arbitrary file into memory. The extra byte distinguishes + "at the cap" from "over it" without reading the rest. """ flags = ( os.O_RDONLY @@ -1008,24 +983,23 @@ def _sweep_orphan_temp_files(root: Path, root_fd: int | None, key: str) -> None: """ Removes temp files a killed reconcile left behind under ``//``. - ``atomic_write`` unlinks its own temp file on any exception, but a ``SIGKILL`` - between the create and the rename leaves one on disk, and nothing else - records that it exists: ``_prune`` walks manifest entries, and an orphan - never has one. The second-order effect is what makes this worth doing — - ``_prune_one``'s ``rmdir`` only succeeds on an empty directory, so a single - orphaned temp pins a skill's directory permanently. + ``atomic_write`` unlinks its own temp file on any exception, but a + ``SIGKILL`` between the create and the rename leaves one behind that nothing + on disk records — ``_prune`` walks manifest entries, and an orphan never has + one. That matters because ``_prune_one``'s ``rmdir`` only succeeds on an + empty directory, so a single orphaned temp pins a skill's directory + permanently. Bounded on every axis, because this is the one place the SDK removes a file the manifest does not list: only inside a directory named by a key that - passes ``_key_rejection_reason``; only names ``safe_fs`` itself recognizes as - its own temp naming for ``SKILL.md``, anchored at both ends, and asked of - ``safe_fs`` rather than re-spelled here so the recognizer cannot drift from - the writer; only regular files; and every removal relative to a descriptor - pinned with ``O_NOFOLLOW`` which is itself opened relative to the root - descriptor, so neither the root nor the skill directory can be swapped for - somewhere else to delete from. It never raises and never aborts the run: the - reconcile itself has succeeded either way, so a sweep that cannot happen is - a warning. + passes ``_key_rejection_reason``; only names ``safe_fs`` recognizes as its + own temp naming for ``SKILL.md``, asked of ``safe_fs`` so the recognizer + cannot drift from the writer; only regular files; and every removal relative + to a descriptor pinned with ``O_NOFOLLOW`` that is itself opened relative to + the root descriptor. + + Never raises and never aborts the run: the reconcile has succeeded either + way, so a sweep that cannot happen is a warning. """ if _key_rejection_reason(key) is not None: return @@ -1087,8 +1061,8 @@ def _write_through_descriptor( invalidated by a swap between here and the rename. The skill directory is created and opened relative to *root_fd*, so the - ``mkdir`` cannot be redirected either — ``mkdir`` follows a symlink at its - parent, which made a root swapped before this point enough to have the + ``mkdir`` cannot be redirected either: ``mkdir`` follows a symlink at its + parent, so against a full path a swapped root is enough to have the directory, and then the file, created outside the root. """ try: @@ -1113,10 +1087,9 @@ def _update_entry( Merges into any existing entry rather than replacing it, so fields written by a future SDK release survive this one's rewrite. - ``sha256`` and ``writtenAt`` are recorded for forensics only: the reconcile - decides currency by hashing the bytes on disk, precisely because the - manifest is untrusted, so neither field is ever read back as a decision - input. + ``sha256`` and ``writtenAt`` are recorded for forensics only. The reconcile + decides currency by hashing the bytes on disk, precisely because the manifest + is untrusted, so neither field is ever read back as a decision input. """ existing = entries.get(relative) entry = dict(existing) if isinstance(existing, dict) else {} @@ -1136,11 +1109,10 @@ def _prune_error(key: str, message: str, version: Any = None) -> ReconcileAction """ A prune refusal. Mirrors ``_write_one``'s local ``failed`` helper. - *version* comes off the manifest, which is untrusted, so it is validated here - rather than at each call site — the same guard the ``removed`` action applies, - so a refusal and a removal report the field identically. - Callers that genuinely do not know a version pass nothing; none of them may - invent one. + *version* comes off the manifest, which is untrusted, so it is validated + here rather than at each call site — the same guard the ``removed`` action + applies, so a refusal and a removal report the field identically. A caller + that does not know a version passes nothing rather than inventing one. """ return ReconcileAction( key=key, @@ -1273,16 +1245,16 @@ def _prune_one( removed_from_disk = True try: # Relative to the root descriptor. rmdir never follows a trailing - # symlink (it fails ENOTDIR) and only ever succeeds on an empty - # directory, so the *key* was never the exposure here — the root - # above it was, since a path-based rmdir re-resolves it and would - # remove an attacker-chosen directory that happened to be empty. + # symlink (it fails ENOTDIR) and only succeeds on an empty + # directory, so the *key* is not the exposure here — the root above + # it is, since a path-based rmdir re-resolves it and could remove an + # attacker-chosen empty directory. if root_fd is not None: os.rmdir(key, dir_fd=root_fd) else: skill_dir.rmdir() except OSError: - pass # the customer keeps their own files here too + pass # the directory is not empty: other files live here too entries.pop(relative, None) diff --git a/packages/client/src/launchdarkly_ai_server/skills_watch.py b/packages/client/src/launchdarkly_ai_server/skills_watch.py index 996a062e..72cad733 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_watch.py +++ b/packages/client/src/launchdarkly_ai_server/skills_watch.py @@ -13,7 +13,7 @@ ``on_unavailable="keep"`` stays the default: an outage must not read as "everything was revoked". A watcher that pruned on a failed retrieval would -convert every transport failure into deletion of a customer's skill files. +convert every transport failure into deletion of the application's skill files. Layering: this module sits *above* ``skills_fs`` and calls ``write_skills`` without modifying it. Nothing in the reconcile, the accessors, or verification @@ -201,8 +201,8 @@ def close(self, timeout: float = 15.0) -> None: Waits out an in-flight reconcile rather than interrupting one, because a reconcile killed between its content writes and its manifest rewrite is - the one case the manifest format has to recover from — worth avoiding when - we control the timing. + the one case the manifest format has to recover from — worth avoiding + where the timing is under the SDK's control. Detaches ``notify`` from the store first, so no further change reaches a watcher that is shutting down and the store no longer holds a reference to diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 291b457c..ba5790f8 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -466,13 +466,11 @@ class Skill: """ The closed set of outcomes ``get_skill_result`` reports. -Alphabetical, as ``reason_code`` is in the integrity log record, so the token -list reads identically in every LaunchDarkly AI SDK. Each token is a distinct -*decision* a caller can make, which is the point of the type: ``absent`` is a -skill the store does not hold, ``integrity_failure`` is content that was -delivered and did not verify, and a caller that wants to fail closed on -suspected tampering while tolerating a merely-absent skill needs the two to be -told apart. +Each token is a distinct *decision* a caller can make, which is the point of the +type: ``absent`` is a skill the store does not hold, ``integrity_failure`` is +content that was delivered and did not verify, and a caller that wants to fail +closed on suspected tampering while tolerating a merely-absent skill needs the +two told apart. - ``ok`` — a verified skill was returned. - ``absent`` — the store answered, and does not hold the key. From e40fbb91d3758dd78a2c15b2db6ee4a20346cd4b Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 16:13:32 -0400 Subject: [PATCH 38/43] docs(client): move the threat model into agents.md, leave invariants in code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The security rationale in `safe_fs.py`, `skills_fs.py` and `skills_core.py` was a threat model written as comments: why a path check cannot hold, what a swapped root redirects, what each flag defends against, what breaks if a bound moves. It is knowledge worth keeping, but a customer reading these modules to learn how to consume the SDK is not its audience, and `agents.md` already had the sections for it — `Security posture`, `Descriptor-pinned filesystem access`, `Telemetry` and `The integrity-failure log record`. So the argument moves there and the code keeps the invariant plus a one-line reason, with the booby-trapped spots still warned about in place: do not "correct" the `os.supports_dir_fd` probe names, do not reach for `errors="surrogatepass"`, do not carry the first verification pass forward, do not drop `sort_keys`, do not widen the adoption exception or the temp sweep. Each module docstring names its invariants and points once at the note, rather than arguing at every call site. `skills_fs.py` goes from 37% comment lines to 33%, `safe_fs.py` from 51% to 46%, against a 22% baseline for the package's other modules. The rest is public API reference, which stays. Also folds two facts from the earlier fixes into the note (the manifest read cap and the two-sided `manifestVersion` bound), and removes the last two internal references, both in the README: a CI-runner aside and a sibling-SDK parity mention that the source sweep did not cover. No behavior change. Co-Authored-By: Claude Opus 5 --- packages/client/README.md | 11 +- packages/client/agents.md | 13 +- .../src/launchdarkly_ai_server/safe_fs.py | 187 ++++++------- .../src/launchdarkly_ai_server/skills_core.py | 114 ++++---- .../src/launchdarkly_ai_server/skills_fdv2.py | 15 +- .../src/launchdarkly_ai_server/skills_fs.py | 251 +++++++----------- 6 files changed, 248 insertions(+), 343 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 4503f53c..9301d320 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -467,10 +467,9 @@ checklist below denies the agent write access to every ancestor. Windows has no taken immediately before each step. That floor is a check-then-use race rather than a closed window: an attacker who already holds **write permission on the managed root** can still win it. Windows reparse-point checks (`GetFileAttributesW` / `FILE_FLAG_OPEN_REPARSE_POINT`) are -deliberately not implemented in this release, and Windows is not a tested platform for it — -neither SDK repository has a Windows CI runner. Treat write permission on the managed root -**or on any directory above it** as the security boundary on every platform, and on Windows as -the *only* one. +deliberately not implemented in this release, and Windows is not a supported or tested +platform for it. Treat write permission on the managed root **or on any directory above it** +as the security boundary on every platform, and on Windows as the *only* one. **One exception, and it is what makes a crashed reconcile recoverable.** A file at a managed path whose bytes are *already byte-identical* to the content LaunchDarkly resolved is @@ -600,8 +599,8 @@ your own helpers: `ReconcileActionKind` (`written` / `updated` / `skipped_curren `removed` / `error`), `OnUnavailable` (`keep` / `raise`), and `SkillOutcomeReason` (`absent` / `integrity_failure` / `ok` / `store_unavailable` / `wrong_version`). -**`write_skills` blocks.** It is `async` for parity with the other accessors and with the -TypeScript SDK, but it awaits nothing: every read, write, `fsync` and rename runs inline, +**`write_skills` blocks.** It is `async` for parity with the other accessors, but it awaits +nothing: every read, write, `fsync` and rename runs inline, so a large reconcile holds the event loop for its duration. Wrap it in `asyncio.to_thread` if that matters. For the same reason `timeout` is checked between steps rather than interrupting one already in progress. Reconcile one root at a time, diff --git a/packages/client/agents.md b/packages/client/agents.md index 49375164..abff7d50 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -431,9 +431,16 @@ Store data is **untrusted input**; the transport is not part of the trust bounda from the writer); only regular files, with the type read off the descriptor; unlinked through the pinned descriptor. It never raises and never aborts a run. - **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed - `entries`, or a `manifestVersion` this release cannot read means no overwrites and no - prunes, brand-new paths may still be written, an `error` action names the manifest, and - the manifest file itself is not rewritten. + `entries`, larger than `_MAX_MANIFEST_BYTES`, or a `manifestVersion` outside + `1 <= v <= MANIFEST_VERSION` means no overwrites and no prunes, brand-new paths may still + be written, an `error` action names the manifest, and the manifest file itself is not + rewritten. The version is bounded on *both* sides: 1 is the first version ever written, so + 0 or a negative is not a manifest this SDK produced, and accepting one would act + destructively on it and then silently rewrite it as version 1. The size bound exists + because the manifest is the one file here whose length no caller can predict, it lives in + a directory the SDK does not own exclusively, and a reconcile must not be the thing that + exhausts the process — every read in `skills_fs` is bounded, and `_read_regular_file` + takes a required `max_bytes` so a new call site cannot opt out by omission. - **An incomplete retrieval suppresses pruning.** Otherwise a transport outage would read as "everything was revoked" and delete the customer's managed files. - **Writes are atomic**: temp file created exclusively in the target's *own* directory, diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index f217ee57..9d3c11f7 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -5,20 +5,18 @@ an attacker may be racing you for" problem, solved once. ``skills_fs.py`` is the only caller today. -A path check is only as good as the last path resolution after it. Every -operation here therefore runs relative to a descriptor pinned to a directory the -caller has already validated, rather than re-resolving a name — which is what -closes the swap window rather than merely narrowing it. - -**This guarantee is POSIX-only.** On POSIX the descriptor walk closes the swap -window. Windows has no ``*at()`` syscall family, so only the per-component -``lstat`` floor runs there, and a floor is a check-then-use race rather than a -closed window; closing it would need reparse-point checks -(``GetFileAttributesW``, or opening with ``FILE_FLAG_OPEN_REPARSE_POINT``), which -this release does not implement. The practical consequence: on Windows, write -permission on the managed root is the *only* boundary, so the privilege-separated -deployment the README describes is the mitigation there rather than merely good -advice. +**The invariant:** every operation runs relative to a descriptor pinned to a +directory the caller already verified, never against a re-resolved path. A path +check is only as good as the last resolution after it. + +**POSIX only.** Windows has no ``*at()`` family, so only the per-component +``lstat`` floor runs there — a check-then-use race rather than a closed window. +Write permission on the managed root is therefore the only boundary on Windows, +which is why the README documents a privilege-separated deployment as the +mitigation rather than as advice. + +The threat model, the platform decision behind it, and what must not be relaxed +are in ``agents.md`` under *Descriptor-pinned filesystem access*. """ from __future__ import annotations @@ -53,24 +51,14 @@ {os.rename, os.open, os.unlink, os.stat, os.mkdir, os.rmdir} ) """ -Whether the ``*at()`` syscall family is available, so every operation under the -managed root can be performed relative to a descriptor pinned to a directory -this module has already verified rather than re-resolved from its path. - -A descriptor refers to the inode that was checked, so replacing -``/`` with a symlink after the check cannot redirect a write or an -unlink out of the root. POSIX has these calls; Windows does not, and there the -per-component ``lstat`` floor applies instead. - -**Do not "correct" the names in this probe.** It deliberately names -``os.rename`` and ``os.stat`` rather than the ``os.replace`` and ``os.lstat`` -this module actually calls, because ``os.supports_dir_fd`` is populated per -underlying syscall: CPython registers ``renameat`` under ``rename`` only and -``fstatat`` under ``stat`` only, even though ``os.replace`` is the same -``renameat``-backed function and ``os.lstat`` is ``fstatat`` with -``AT_SYMLINK_NOFOLLOW``, and both accept the descriptor keywords wherever their -advertised twin does. Probing the names this module calls would report -"unsupported" on every POSIX platform and silently disable the defense. +Whether the ``*at()`` syscall family is available. Gates every descriptor-pinned +operation in this module; ``False`` falls back to the ``lstat`` floor. + +**Do not "correct" the names in this probe.** ``os.supports_dir_fd`` is +populated per underlying syscall, and CPython registers ``renameat`` under +``rename`` only and ``fstatat`` under ``stat`` only — so probing the +``os.replace`` and ``os.lstat`` this module actually calls reports +"unsupported" on every POSIX platform and silently disables the defense. """ @@ -78,12 +66,10 @@ def _at(directory: Path, dir_fd: int | None) -> str | Path: """ What to name *directory* by, given a descriptor for its parent. - With a *dir_fd* the call must pass the bare final component, so the kernel - resolves it inside the pinned parent and no ancestor is re-resolved from its - path; without one the full path is the only thing there is to pass. Spelled - once because every operation in this module that accepts a parent - descriptor has to make the same choice, and one call site left on the full - path would silently re-open the window the descriptor closes. + With a *dir_fd*, the bare final component, so the kernel resolves it inside + the pinned parent; without one, the full path. Spelled once because a single + call site left on the full path would silently reopen the window the + descriptor closes. """ return directory.name if dir_fd is not None else directory @@ -94,23 +80,18 @@ def open_directory_nofollow( """ Opens *directory* without following a final symlink, and pins it. - *dir_fd* is a descriptor for the *parent*, and passing one is what extends - the guarantee past the final component: ``O_NOFOLLOW`` refuses a link at - *directory* itself, but every ancestor above it is re-resolved from its path - on each open, so a parent swapped for a symlink after it was checked - redirects the open. Given a parent descriptor the bare name is resolved - inside the inode that was checked instead, and there is nothing left to swap. - - On a platform without the ``*at()`` family (Windows) this returns ``None`` - after verifying via ``lstat`` that the path is a real, non-symlink - directory — the per-component floor. It must not attempt the descriptor - open there: ``os.open`` goes through the CRT on Windows, which cannot open - a directory at all, so the descriptor path would fail every operation - rather than fall back. - - Raises ``ValueError`` when the path will not open (or inspect) as a real - directory — the caller reports that as a refusal rather than letting it - escape. + *dir_fd* is a descriptor for the *parent*. Passing one extends the guarantee + past the final component: ``O_NOFOLLOW`` refuses a link at *directory* + itself, but without a parent descriptor every ancestor is re-resolved on + each open. + + Returns ``None`` where the ``*at()`` family is absent, after an ``lstat`` + check for a real non-symlink directory. It must not attempt the descriptor + open there: ``os.open`` cannot open a directory on Windows, so that path + would fail every operation rather than fall back. + + Raises ``ValueError`` when the path will not open, or inspect, as a real + directory. """ if not SUPPORTS_DIR_FD: try: @@ -147,21 +128,18 @@ def open_or_create_directory( """ Creates *directory* if absent and returns a descriptor pinned to it. - ``Path.mkdir(exist_ok=True)`` treats an existing symlink-to-directory as - "already there", which would re-open the very hole the caller's ``lstat`` - check just closed. ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` - path does not: a link reports as a link, and is refused. + ``os.mkdir`` plus an ``lstat`` on the ``FileExistsError`` path, never + ``Path.mkdir(exist_ok=True)``: that accepts an existing + symlink-to-directory as "already there", reopening the hole the caller's + check just closed. - *dir_fd* is a descriptor for the parent, as in ``open_directory_nofollow``, - and the ``mkdir`` needs it every bit as much as the open does: ``mkdir`` - follows a symlink at the parent, so a create issued against the full path is - how a directory gets made — and then written into — outside the root. + *dir_fd* is a descriptor for the parent, and the ``mkdir`` needs it as much + as the open does — ``mkdir`` follows a symlink at its parent, so a create + against the full path is how a directory gets made, and then written into, + outside the root. """ - # As in ``atomic_write``: a parent descriptor is only usable where the - # ``*at()`` family is. ``open_directory_nofollow`` returns ``None`` on the - # platforms without it, so no caller here can hold one — stated rather than - # left to that invariant, because the ``mkdir`` below would otherwise raise - # instead of taking the full-path floor the openers fall back to. + # A parent descriptor is only usable where the ``*at()`` family is, and the + # mkdir below would raise rather than take the floor without this. if not SUPPORTS_DIR_FD: dir_fd = None try: @@ -189,13 +167,12 @@ def pinned_directory( Holds *directory* pinned for the duration of the block, then releases it. Yields what the two openers above return — a descriptor, or ``None`` on the - ``lstat`` floor — so the caller states the platform split once, as - ``if dir_fd is not None``, and cannot forget the ``os.close``. Raises - ``ValueError`` for a directory that will not pin, exactly as they do. + ``lstat`` floor — so a caller states the platform split once and cannot + forget the ``os.close``. Raises ``ValueError`` for a directory that will not + pin. - *dir_fd* is a descriptor for the parent and is passed straight through. Note - which descriptor is which: the one passed *in* pins the parent, and the one - yielded pins *directory* itself. + Note which descriptor is which: *dir_fd* pins the parent, and the yielded + one pins *directory* itself. """ dir_fd = ( open_or_create_directory(directory, dir_fd=dir_fd) @@ -223,18 +200,15 @@ def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: """ Removes ``/``, refusing to follow a symlink at *name*. - The mirror of ``atomic_write``, and descriptor-relative for the same reason: - ``unlink`` never follows a *trailing* symlink, but it does resolve the - directory above it, so a ```` swapped for a symlink after the - caller's checks would otherwise turn this into a delete of an - attacker-chosen file. Given a *dir_fd* the probe and the unlink both run - against it; without one the identical sequence runs against full paths. - - Raises ``SymlinkRefused`` when *name* is a symlink. Note that this refuses - rather than removes: ``unlink`` would happily delete the link itself, but a - link where this SDK expects its own file means the state on disk is not what - the manifest describes, and that is the caller's to report rather than to - tidy away. + Descriptor-relative for the same reason as ``atomic_write``: ``unlink`` + never follows a *trailing* symlink, but it does resolve the directory above + it, so a swapped ```` would turn this into a delete of an + attacker-chosen file. + + Raises ``SymlinkRefused`` when *name* is a symlink — refusing rather than + removing, because a link where the SDK expects its own file means the state + on disk is not what the manifest describes, and that is the caller's to + report rather than to tidy away. """ if dir_fd is None: # No ``*at()`` family: the trailing-symlink check and the unlink are both @@ -260,12 +234,9 @@ def unlink_file(directory: Path, name: str, *, dir_fd: int | None) -> None: """Bytes of randomness in a temp name, as ``secrets.token_hex`` takes them.""" _TEMP_TOKEN_PATTERN = re.compile( - # Two producers, one recognizer. The descriptor path below names its temp - # file with ``secrets.token_hex(_TEMP_TOKEN_BYTES)`` — twice that many - # lowercase hex characters. The fallback path hands naming to - # ``tempfile.mkstemp``, whose sequence is eight characters drawn from - # ``[a-z0-9_]``. Matched with ``fullmatch``, which anchors both branches at - # both ends, so nothing longer or otherwise-shaped is ever recognized. + # Two producers, one recognizer: ``secrets.token_hex`` on the descriptor + # path, ``tempfile.mkstemp``'s eight ``[a-z0-9_]`` characters on the + # fallback. Used with ``fullmatch``, so both branches are anchored. rf"[0-9a-f]{{{_TEMP_TOKEN_BYTES * 2}}}|[a-z0-9_]{{8}}" ) @@ -274,10 +245,9 @@ def temp_name_prefix(name: str) -> str: """ The prefix every temp file for *name* is created under. - Spelled once because two callers need to agree on it: ``atomic_write`` - creates the name, and a caller sweeping orphaned temp files left by a crash - has to recognize it. A copy of the format string in the sweeper would be a - copy that can drift out of step with the writer. + Spelled once because two callers must agree: ``atomic_write`` creates the + name and the orphan sweep recognizes it, and a second copy of the format + would drift from the writer. """ return f".{name}." @@ -286,12 +256,9 @@ def is_temp_name(candidate: str, name: str) -> bool: """ Whether *candidate* is a name this module could have created for *name*. - The recognizer for the orphan sweep: ``atomic_write`` unlinks its temp file - on any exception, but a ``SIGKILL`` between the create and the rename leaves - it behind, and nothing else on disk records that it exists. Deliberately - narrow — prefix, random token, and suffix must all match, with nothing - before or after — because the only thing a caller does with a ``True`` here - is delete the file. + Deliberately narrow — prefix, random token and suffix must all match, with + nothing before or after — because the only thing a caller does with a + ``True`` here is delete the file. """ prefix = temp_name_prefix(name) if not candidate.startswith(prefix) or not candidate.endswith(_TEMP_SUFFIX): @@ -306,8 +273,7 @@ def _mkstemp_at(dir_fd: int, prefix: str) -> tuple[int, str]: ``tempfile`` has no ``dir_fd`` form, so this reproduces the part that matters: ``O_CREAT | O_EXCL`` against an unpredictable name, retried on - collision, so an existing temp path is never reused and a planted one is - never written through. + collision, so a planted temp path is never written through. """ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) for _ in range(tempfile.TMP_MAX): @@ -327,15 +293,14 @@ def atomic_write( observable. The temp file is created exclusively in the target's *own* directory — one - anywhere else would make the rename cross-device, and therefore not atomic — + anywhere else would make the rename cross-device, and so not atomic — written, fsynced, renamed over the target, and the directory fsynced so the - rename itself survives a crash. Mode is set explicitly rather than left to - the process umask, and the execute bit is never set. + rename survives a crash. Mode is set explicitly rather than left to the + umask, and the execute bit is never set. - Given a *dir_fd* on a platform with the ``*at()`` family, every one of those - steps runs relative to that descriptor and both names are bare filenames. - Without one (Windows) the identical sequence runs against full paths, which - is the per-component ``lstat`` floor. + Given a *dir_fd*, every one of those steps runs relative to that descriptor + and both names are bare filenames; without one, the identical sequence runs + against full paths. ``os.replace`` is the one and only rename call site. ``os.rename`` must not be substituted for it: only ``os.replace`` has defined overwrite semantics diff --git a/packages/client/src/launchdarkly_ai_server/skills_core.py b/packages/client/src/launchdarkly_ai_server/skills_core.py index 36256207..8da0ba8f 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_core.py +++ b/packages/client/src/launchdarkly_ai_server/skills_core.py @@ -6,14 +6,17 @@ this module imports neither ``skills`` nor ``skills_fs``. It holds the store interface and the configured store, the telemetry emitter, -integrity verification, and store resolution. Each lives here in one copy so -that the accessors and the materialization path cannot disagree: about whether a -store is configured, about which signals exist, about what verification accepts, -or about how a raising store is handled. - -Everything the store hands back is untrusted input; the transport is not part of -the trust boundary. Key, version, size, and content hash are revalidated here on -every pass. +integrity verification, and store resolution. Each lives here in one copy so the +accessors and the materialization path cannot disagree: about whether a store is +configured, about which signals exist, about what verification accepts, or about +how a raising store is handled. + +**Everything a store hands back is untrusted input**; the transport is not part +of the trust boundary. Key, version, size, and content hash are revalidated here +on every pass, and no value off the wire is echoed into a signal or a log line +without a shape check. The reasoning, and the signal and log-record contracts +these must satisfy, are in ``agents.md`` under *Security posture*, *Telemetry* +and *The integrity-failure log record*. """ from __future__ import annotations @@ -73,13 +76,10 @@ Stable event identity for the local integrity-failure log record. A compatibility surface, not an implementation detail: this is the string a SIEM -matches on, so it must never be renamed. - -It appears verbatim **in the message text**, not only in ``extra``. Severity -alone cannot discriminate — ``list_raw_objects`` and ``resolve_from_store`` in -this module also log ERROR when a store raises — and the stdlib's default -formatter drops ``extra`` entirely, so under a plain ``logging.basicConfig()`` -an ``extra``-only record would be invisible. +matches on, so it must never be renamed. It appears verbatim in the message +text, not only in ``extra``, because the stdlib's default formatter drops +``extra`` and severity alone cannot discriminate — a raising store logs ERROR +from this module too. """ _ACTION_WITHHELD = "withheld" @@ -306,23 +306,17 @@ def record_integrity_failure( Carries hashes and byte counts only — the skill body never appears in a signal, a log line, or an error message. - The two are deliberately different sizes. The signal is product telemetry: - no-op by default, with a fixed property set. The log record is the - application's own detection path — the only one that works when telemetry is - off — so it additionally carries the stable event name, the action taken, the - human-readable reason, and the machine-parseable ``reason_code``. - - The record is emitted in two forms because neither alone is enough: the - message text carries ``INTEGRITY_FAILURE_EVENT`` followed by compact JSON, so - it survives ``logging.basicConfig()`` and is greppable and ``jq``-able under - any handler configuration, and ``extra["ld_skills"]`` carries the same - mapping unflattened for a structured handler that would rather not reparse. - """ - # Both of these come off the wire, so neither may be echoed verbatim: a store - # that set contentHash (or key) to the skill body would otherwise publish the - # body itself. Shape-check, then redact. Every field the log record adds on - # top is either a literal or SDK-authored, so the record introduces no new - # untrusted value — anything added later needs this same treatment. + The signal is product telemetry: no-op by default, with a fixed property + set. The log record is the application's own detection path — the only one + that works when telemetry is off — so it also carries the stable event name, + the action taken, the reason, and the machine-parseable ``reason_code``, in + both the message text and ``extra["ld_skills"]``. Neither form alone + survives every handler configuration; ``agents.md`` states the contract. + """ + # Key and expected hash come off the wire, so neither may be echoed + # verbatim: a store could put the skill body in either. Shape-check, then + # redact. Every field added below is a literal or SDK-authored; anything + # added later needs this same treatment. safe_key = skill_key if is_valid_skill_key(skill_key) else "" properties: dict[str, Any] = {"skill_key": safe_key, "language": _LANGUAGE} if is_valid_skill_version(version): @@ -337,9 +331,8 @@ def record_integrity_failure( properties["observed_hash"] = observed_hash # Spread the signal's properties rather than rebuilding them, so the record - # cannot drift from the signal on the fields they share — in particular on - # which of them are redacted and which are omitted. Absent optional fields - # stay absent; the record never carries a null. + # cannot drift from the signal on which fields are redacted or omitted. + # Absent optional fields stay absent; the record never carries a null. record: dict[str, Any] = { "event": INTEGRITY_FAILURE_EVENT, "action": _ACTION_WITHHELD, @@ -347,10 +340,9 @@ def record_integrity_failure( "reason": reason, **properties, } - # ``sort_keys`` is part of the record's format rather than cosmetic: it is - # what makes the serialized line stable for a given input, so a detection - # rule can match on it. Do not drop it, and do not reorder the keys above - # expecting the output to follow. + # ``sort_keys`` is part of the record's format, not cosmetic: it is what + # makes the line stable for a given input. Do not drop it, and do not + # reorder the keys above expecting the output to follow. logger.error( "%s %s", INTEGRITY_FAILURE_EVENT, @@ -364,10 +356,10 @@ def record_materialized( skill_key: str, content_bytes: int, content_hash: str, reconcile_action: str ) -> None: """ - Records a materialization. Deliberately carries no ``target_path`` and no - filesystem path of any kind — the same reasoning that keeps the skill body - out of telemetry keeps the application's directory layout out. Paths live in the - returned ``ReconcileReport``, which is user-facing API rather than telemetry. + Records a materialization. Carries no filesystem path of any kind: the same + reasoning that keeps the skill body out of telemetry keeps the directory + layout out. Paths live in the returned ``ReconcileReport`` instead, which is + API rather than telemetry. """ emit( _SIGNAL_MATERIALIZED, @@ -431,23 +423,20 @@ def verified_bytes( """ The whole content half of integrity verification: encode, size, hash. - Accepts either shape content legitimately arrives in. Wire-shaped ``str`` - input — a raw store object's JSON string — is UTF-8 encoded here, once, and - this is the only place that encode happens. ``bytes`` input is an already - verified ``Skill.content`` being re-verified, and is hashed directly: those - bytes are the verbatim value, so re-encoding does not apply. + Accepts either shape content arrives in. A wire-shaped ``str`` is UTF-8 + encoded here, once — the only place that encode happens. ``bytes`` is an + already-verified ``Skill.content`` being re-verified, and is hashed directly. Returns the verbatim bytes and their locally computed sha256, or a - human-readable reason — having already recorded the integrity signal, so the - signal does not depend on which caller noticed. The hash handed back is - always the one computed here, never the caller's expected value, which keeps - an untrusted string out of ``Skill``. - - This runs twice per skill by design: once at the accessor boundary, and again - immediately before a write, because a ``Skill`` can also be constructed - directly by a caller. The second pass re-hashes bytes the first pass already - hashed, which is negligible next to the write it guards — and carrying the - first pass's verdict forward would put a "trust the value computed upstream" + human-readable reason, having already recorded the integrity signal so it + does not depend on which caller noticed. The hash handed back is always the + one computed here, never the caller's expected value, which keeps an + untrusted string out of ``Skill``. + + Runs twice per skill by design — at the accessor boundary, and again + immediately before a write, since a ``Skill`` can also be constructed + directly by a caller. Do not optimise the second pass away by carrying the + first one's verdict forward: that puts a "trust the value computed upstream" branch inside the one function whose job is not to. """ if isinstance(content, bytes): @@ -456,11 +445,10 @@ def verified_bytes( try: encoded = content.encode("utf-8") except UnicodeEncodeError: - # json.loads turns a "\ud800" escape into an unpaired surrogate, which - # has no UTF-8 encoding. There are no bytes the server could have - # hashed, so this is not authentic content. Never use - # errors="surrogatepass" here: that would fabricate bytes and could - # satisfy the hash comparison. + # json.loads turns a "\ud800" escape into an unpaired surrogate, + # which has no UTF-8 encoding — so there are no bytes the server + # could have hashed. Never reach for errors="surrogatepass": it + # would fabricate bytes that could satisfy the hash comparison. reason = "content is not encodable as UTF-8" record_integrity_failure( key, diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index e0934470..6a964eae 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -148,7 +148,8 @@ def _require_server_side_credential(sdk_key: str) -> None: Skill content is customer-confidential, and payload assignment is shared across credential types, so a client-side credential may well *succeed* - against these endpoints. Raises rather than logs: a store built on the wrong + against these endpoints — which is why this refuses rather than relying on + the server to. Raises rather than logs: a store built on the wrong credential should not exist. """ if not isinstance(sdk_key, str) or not sdk_key.strip(): @@ -512,13 +513,11 @@ class _ProtocolReader: **The first payload intent is read, and is taken to be the skill payload.** Delivery provides one payload per credential and the protocol requires a - client to ignore all but the first intent, so ``payloads[0]`` is both what - arrives and what the protocol says to read. Should that ever widen, an - ``xfer-full`` for another payload would empty the skill set and the next - ``payload-transferred`` would publish it empty — with pruning on, the - difference between a reconcile and deleting the application's files. So this - layer learns which payload skills arrive on and declines to apply a transfer - of any other, warning once and counting it. The residual case is the first + client to ignore all but the first intent. Should that ever widen, an + ``xfer-full`` for another payload would publish the skill set empty — with + pruning on, the difference between a reconcile and deleting the + application's files. So this layer learns which payload skills arrive on and + declines to apply a transfer of any other. The residual case is the first transfer of a connection, where there is nothing to compare against yet. """ diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 99fd8f48..6ec85fa5 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -2,23 +2,21 @@ Agent Skills — filesystem materialization. This is the part that writes to disk. Everything here takes already-verified -content and reconciles it against a managed root, while ``skills.py`` owns -retrieval and verification and knows nothing about the filesystem. The -dependency runs one way only, and the descriptor-pinned primitives every -destructive step goes through live in ``safe_fs.py``. - -The managed root is pinned to a descriptor once per reconcile and every -destructive operation runs relative to it, so no ancestor of the root is ever -re-resolved from its path mid-run. The path checks still run, but they are -defense in depth rather than the boundary: a check on a path is only as good as -the last resolution of that path after it, and the descriptor is what makes -there be no later resolution. - -The reconcile is manifest-driven and fails closed. Destructive operations only -ever touch paths ``/.launchdarkly-skills.json`` records under a matching -key; a corrupt manifest suppresses every destructive action; and an incomplete -retrieval suppresses pruning. Content is re-verified immediately before the -write, because a ``Skill`` can also be constructed directly by a caller. +content and reconciles it against a managed root; ``skills.py`` owns retrieval +and verification and knows nothing about the filesystem. The descriptor-pinned +primitives every destructive step goes through live in ``safe_fs.py``. + +**The invariants.** The managed root is pinned to a descriptor once per +reconcile and every destructive operation runs relative to it. Destructive +operations only ever touch paths ``/.launchdarkly-skills.json`` records +under a matching key. A corrupt manifest suppresses every destructive action, +and an incomplete retrieval suppresses pruning. Content is re-verified +immediately before the write. The path checks still run, but as defense in +depth rather than as the boundary. + +None of these may be relaxed. The threat model behind each — and what breaks if +one moves — is in ``agents.md`` under *Security posture* and *Descriptor-pinned +filesystem access*. """ from __future__ import annotations @@ -108,11 +106,10 @@ _MAX_PATH_COMPONENT_BYTES = 255 """ -NAME_MAX on Linux and macOS, and the component limit on Windows. A skill key -becomes a single directory name, and the data model permits keys up to 256 -characters — one byte longer than any of those filesystems can represent. Such a -key is rejected before any filesystem call so the caller gets a reported action -rather than an ENAMETOOLONG escaping from a stat deep inside the reconcile. +NAME_MAX on Linux and macOS, and the component limit on Windows. The data model +permits keys one byte longer than any of those can represent, so an over-long +key is rejected before any filesystem call — a reported action rather than an +ENAMETOOLONG from a stat deep inside the reconcile. """ @@ -122,15 +119,13 @@ | {f"lpt{digit}" for digit in range(1, 10)} ) """ -The 22 MS-DOS device names Windows still reserves, which cannot be directory -names there. The key grammar admits every one of them, so a skill named ``con`` -would reconcile on Linux and fail on Windows. Rejected here instead, on every -platform, so the on-disk result never depends on which OS ran the write. - -The bare names are the whole set: no suffix stripping is needed because the key -grammar admits no ``.``, so ``con.txt`` is unreachable, and ``CONIN$`` / -``CONOUT$`` are unreachable for want of a ``$``; no case folding is needed -because the grammar is lowercase-only. ``com0`` and ``lpt0`` are deliberately +The 22 MS-DOS device names Windows reserves, which cannot be directory names +there. Rejected on every platform, so the on-disk result never depends on which +OS ran the write. + +The bare names are the whole set: the key grammar admits no ``.`` or ``$`` and +is lowercase-only, so ``con.txt`` and ``CONIN$`` are unreachable and no suffix +stripping or case folding is needed. ``com0`` and ``lpt0`` are deliberately absent — those are not reserved. """ @@ -179,12 +174,9 @@ async def write_skills( ``ValueError`` for a caller error such as an unusable root. The root is opened once, up front, and that descriptor is held until the - call returns: every directory created, file written and file removed under - it is resolved relative to the descriptor rather than from the root's path, - so a root replaced after validation is refused rather than followed. The - manifest is read through it too, so the record that authorizes those - removals comes from inside the pinned root rather than from wherever the - root's path led by the time it was read. This guarantee is POSIX-only; see + call returns — including for the manifest read, so the record that + authorizes a removal comes from inside the pinned root. A root replaced + after validation is refused rather than followed. POSIX only; see ``safe_fs``. **This call performs synchronous filesystem I/O and does not yield.** It is @@ -213,17 +205,9 @@ async def write_skills( deadline = time.monotonic() + timeout root_path = _resolve_root(root) - # The root is pinned once, here, and held for the whole reconcile. Every - # destructive step below runs relative to this descriptor, which is what - # makes _resolve_root's validation mean something afterwards: O_NOFOLLOW on - # a per-operation open of / guards only that final component, so - # the root and its ancestors were re-resolved on every such open and a root - # renamed aside and replaced with a symlink after validation redirected the - # open — and every descriptor-relative step behind it — out of the root. - # A descriptor names the inode that was checked, so there is nothing left to - # re-resolve and nothing left to swap. On a platform with no *at() family - # this is None and the per-component lstat floor applies instead, as it does - # throughout safe_fs. + # Pinned once and held for the whole reconcile, which is what makes + # _resolve_root's validation mean anything afterwards. None where the *at() + # family is absent, and the lstat floor applies instead. try: root_fd = open_directory_nofollow(root_path) except ValueError as exc: @@ -601,14 +585,12 @@ def _resolve_root(root: str | os.PathLike[str]) -> Path: Resolves the managed root once, up front. An unusable root is a caller error rather than a per-skill outcome, so this - raises. Only the leaf directory is ever created — recursively creating + raises. Only the leaf directory is ever created: recursively creating missing ancestors would let a typo scatter a directory tree. - What this establishes is that the root was usable *at this instant*, which - is a caller-error check and not a security boundary — the path it returns - can be replaced the moment it returns. ``write_skills`` pins the returned - path to a descriptor immediately afterwards, and that is what carries the - guarantee for the rest of the run. + This is a caller-error check, not a security boundary — it establishes only + that the root was usable at this instant. ``write_skills`` pins the returned + path immediately afterwards, and that is what carries the guarantee. """ path = Path(os.fspath(root)) @@ -666,16 +648,11 @@ def _load_manifest( An absent manifest is not corrupt — that is simply a fresh root. - Read relative to the root descriptor the caller holds, for the same reason - every destructive step is: this file decides which of the application's - files the SDK may overwrite and delete, so the pin has to cover the decision - as well as the actions. Read by path, a root swapped after the pin would - hand the run somebody else's entries while every consequence landed in the - real root. - - That single open also means absence is ``ENOENT`` on the read itself, and - that a symlink or a FIFO wearing the manifest's name is refused as - corruption rather than followed or waited on. + Read relative to the root descriptor, because this file decides which files + the SDK may overwrite and delete: the pin has to cover the decision as well + as the actions. That single open also makes absence ``ENOENT`` on the read + itself, and refuses a symlink or FIFO wearing the manifest's name as + corruption rather than following or waiting on it. """ fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} @@ -757,24 +734,18 @@ def _unsafe_path_reason( The path defenses, in one place. Returns why ``//SKILL.md`` must not be touched, or ``None``. - Shared by the write and prune paths, and not to be relaxed in either: - maintaining these twice is how they drift. - - These are defense in depth and not the boundary. Every one of them inspects - a path, so each is a check-then-use against anything that can rename a - component of that path — which is precisely why the destructive steps below - run relative to the root descriptor instead. They stay because they turn a - hostile layout into a reported refusal rather than a failed syscall, and - because they are the whole defense on a platform with no ``*at()`` family. - - *require_directory* is the one genuine difference between the two callers. A - write needs a real directory to write into. A prune only needs to not follow - a link — an entry whose directory has been replaced by a plain file has - already lost the file this SDK owned, so reporting ``removed`` is what lets - the stale manifest entry be dropped rather than pinned forever. - - Note that the containment check is unconditional even though ``skill_dir`` - may not exist yet: ``realpath`` resolves the existing prefix and appends the + Shared by the write and prune paths so the two cannot drift, and not to be + relaxed in either. + + Defense in depth rather than the boundary: each of these inspects a path, so + each is a check-then-use. They stay because they turn a hostile layout into + a reported refusal rather than a failed syscall, and because they are the + whole defense where the ``*at()`` family is absent. + + *require_directory* is the only difference between the two callers: a write + needs a real directory to write into, while a prune only needs to not follow + a link. The containment check is unconditional even when ``skill_dir`` does + not exist yet — ``realpath`` resolves the existing prefix and appends the rest, so a fresh key under a valid root passes. """ if skill_dir.is_symlink(): @@ -792,14 +763,12 @@ def _key_rejection_reason(key: Any) -> str | None: """ Why *key* must not become a directory name under the managed root, or ``None``. - Re-validated locally whatever any upstream layer already did, and before - any filesystem call, because a key becomes a path component. Shared by the - write and prune paths so the two cannot disagree about which keys this SDK - could own, and not to be relaxed in either. + Re-validated locally whatever any upstream layer already did, and before any + filesystem call, because a key becomes a path component. Shared by the write + and prune paths, and not to be relaxed in either. - ``key.encode`` is safe here only because it runs *after* the pattern check: - the key grammar admits no surrogate, so there is no unencodable key left to - raise on. Do not reorder these two. + ``key.encode`` below is safe only because it runs *after* the pattern check, + which admits no surrogate. Do not reorder the two. """ if not is_valid_skill_key(key): return f"{key!r} is not a valid skill key: it {skill_key_rejection_reason(key)}" @@ -813,10 +782,9 @@ def _key_rejection_reason(key: Any) -> str | None: f"{_MAX_PATH_COMPONENT_BYTES}-byte limit for a single directory name" ) # Checked here rather than in the grammar, for the same reason as the byte - # bound above: rejecting these in the grammar would fail a whole AI Config - # over one skill, and would shrink ``skill_refs`` — which is what authorizes - # a prune, so a Windows-only constraint would delete the skill's file on - # Linux. + # bound above. See agents.md: a grammar-level rejection would fail a whole + # AI Config over one skill, and would shrink skill_refs, which is what + # authorizes a prune. if key in _WINDOWS_RESERVED_NAMES: return ( f"skill key '{key}' is a name Windows reserves for a device and " @@ -876,12 +844,10 @@ def failed(message: str) -> ReconcileAction: if exists: # Hash first, and decide from the bytes. The manifest check below is # what protects a file the SDK did not write, but on its own it also - # refuses one the SDK wrote and was killed before recording: the - # reconcile writes every skill and only then rewrites the manifest, so a - # crash in that window leaves a managed path with no entry, and every - # later reconcile would take the refusal branch forever. Comparing the - # bytes separates the two cases without weakening anything, since only - # content byte-identical to what LaunchDarkly resolved is adopted. + # refuses one the SDK wrote and was killed before recording it, wedging + # every later reconcile. Comparing the bytes separates those two cases, + # and only content byte-identical to what LaunchDarkly resolved is + # adopted. This exception must not be widened — see agents.md. try: on_disk = _read_regular_file(target, max_bytes=len(encoded)) except OSError as exc: @@ -897,12 +863,10 @@ def failed(message: str) -> ReconcileAction: return failed(f"'{relative}' could not be read: {exc}") if hashlib.sha256(on_disk).hexdigest() == content_hash: - # ``skipped_current`` rather than a new action kind: its meaning is - # that the bytes on disk already are the resolved content, which is - # as true for an adopted file as for one the SDK wrote and recorded. - # Adoption adds a manifest entry, so the file becomes prunable - # later — correct, since a prune then removes content LaunchDarkly - # delivered anyway. + # ``skipped_current`` rather than a new action kind: the bytes on + # disk already are the resolved content, as true for an adopted file + # as for one the SDK wrote. Adoption records a manifest entry, so + # the file becomes prunable later. _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, "skipped_current") return ReconcileAction( @@ -941,20 +905,16 @@ def _read_regular_file( """ Reads *target*, refusing anything that is not a regular file. - Each flag earns its place. ``O_NONBLOCK`` because opening a FIFO with no - writer blocks forever, so a managed file swapped for one would hang the - reconcile and the event loop with it (it is a no-op for regular files). - ``O_NOFOLLOW`` to refuse a trailing symlink. ``O_BINARY`` to keep the bytes - verbatim: 0 on POSIX, but without it a Windows descriptor translates CRLF on - read, which would fail the hash comparison against current content. The type - check reads ``fstat`` on the descriptor, never the path. + Each flag earns its place. ``O_NONBLOCK``: opening a FIFO with no writer + blocks forever, so a managed file swapped for one would hang the reconcile + and the event loop with it (a no-op for regular files). ``O_NOFOLLOW``: no + trailing symlink. ``O_BINARY``: 0 on POSIX, but without it a Windows + descriptor translates CRLF and the bytes stop being verbatim. The type check + reads ``fstat`` on the descriptor, never the path. - With *dir_fd*, *target* is a bare component resolved against that descriptor - rather than a path the kernel walks from the root down. - - ``max_bytes`` bounds the read at ``max_bytes + 1`` bytes, so no read here can - be made to pull an arbitrary file into memory. The extra byte distinguishes - "at the cap" from "over it" without reading the rest. + ``max_bytes`` is required, not optional, so a new call site cannot pull an + arbitrary file into memory by omission. The read stops at ``max_bytes + 1``, + the extra byte distinguishing "at the cap" from "over it". """ flags = ( os.O_RDONLY @@ -984,19 +944,15 @@ def _sweep_orphan_temp_files(root: Path, root_fd: int | None, key: str) -> None: Removes temp files a killed reconcile left behind under ``//``. ``atomic_write`` unlinks its own temp file on any exception, but a - ``SIGKILL`` between the create and the rename leaves one behind that nothing - on disk records — ``_prune`` walks manifest entries, and an orphan never has - one. That matters because ``_prune_one``'s ``rmdir`` only succeeds on an - empty directory, so a single orphaned temp pins a skill's directory - permanently. - - Bounded on every axis, because this is the one place the SDK removes a file - the manifest does not list: only inside a directory named by a key that - passes ``_key_rejection_reason``; only names ``safe_fs`` recognizes as its - own temp naming for ``SKILL.md``, asked of ``safe_fs`` so the recognizer - cannot drift from the writer; only regular files; and every removal relative - to a descriptor pinned with ``O_NOFOLLOW`` that is itself opened relative to - the root descriptor. + ``SIGKILL`` between the create and the rename leaves one behind that no + manifest entry records — and ``_prune_one``'s ``rmdir`` only succeeds on an + empty directory, so one orphan pins a skill's directory permanently. + + This is the one place the SDK removes a file the manifest does not list, so + it is bounded on every axis: inside ``//`` only, for a key that + passes ``_key_rejection_reason``; only names ``safe_fs`` recognizes, asked + of ``safe_fs`` so the recognizer cannot drift from the writer; only regular + files; every removal descriptor-relative. Never widen these — see agents.md. Never raises and never aborts the run: the reconcile has succeeded either way, so a sweep that cannot happen is a warning. @@ -1056,14 +1012,10 @@ def _write_through_descriptor( Performs the write itself. Returns a failure reason, or ``None`` on success. Split out of ``_write_one`` because everything above it decides *whether* to - write and this decides nothing: the directory is pinned to a descriptor and - every remaining step is relative to it, so none of the checks above can be - invalidated by a swap between here and the rename. - - The skill directory is created and opened relative to *root_fd*, so the - ``mkdir`` cannot be redirected either: ``mkdir`` follows a symlink at its - parent, so against a full path a swapped root is enough to have the - directory, and then the file, created outside the root. + write and this decides nothing: the directory is pinned and every remaining + step is relative to that descriptor, so no check above can be invalidated by + a swap between here and the rename. The ``mkdir`` is relative to *root_fd* + for the same reason — it follows a symlink at its parent. """ try: with pinned_directory(skill_dir, create=True, dir_fd=root_fd) as dir_fd: @@ -1195,14 +1147,11 @@ def _unlink_through_descriptor( """ Performs the removal itself. Returns a failure reason, or ``None`` on success. - The mirror of ``_write_through_descriptor``, and split out for the same - reason: everything above it decides *whether* to remove, and this decides - nothing. The directory is pinned before the unlink because unlink never - follows a trailing symlink but does resolve the directory above it, so a - ``/`` swapped for a symlink between the checks and here would - otherwise delete a file outside the root — and the pin is taken relative to - *root_fd* for the same reason one level up, since opening ``/`` - by path re-resolves ```` and a swap there redirects the whole removal. + The mirror of ``_write_through_descriptor``, split out for the same reason. + The directory is pinned before the unlink because unlink never follows a + trailing symlink but does resolve the directory above it, and the pin is + taken relative to *root_fd* because opening ``/`` by path + re-resolves ````. """ try: with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: @@ -1244,11 +1193,9 @@ def _prune_one( return _prune_error(key, failure, version) removed_from_disk = True try: - # Relative to the root descriptor. rmdir never follows a trailing - # symlink (it fails ENOTDIR) and only succeeds on an empty - # directory, so the *key* is not the exposure here — the root above - # it is, since a path-based rmdir re-resolves it and could remove an - # attacker-chosen empty directory. + # Relative to the root descriptor: rmdir is safe at the key (it + # fails ENOTDIR on a symlink and needs an empty directory), but a + # path-based call re-resolves the root above it. if root_fd is not None: os.rmdir(key, dir_fd=root_fd) else: From 4bc12c0948faab4204001ce7996b986cc2e3085a Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Tue, 15 Sep 2026 16:13:32 -0400 Subject: [PATCH 39/43] test(client): make the no-track assertions order-independent `test_no_ld_track_calls_from_write_skills` and `test_no_ld_track_calls_from_accessors` assert that `track` was never called, but `init_client` flushes `$ld:ai:sdk:info` on the first client of a process and `reset_skill_state` does not reset that reporting state. Whether the event fires therefore depends on what ran earlier in the session: both tests passed in a full run and failed when their file ran alone, which also makes them fragile under any test sharding. Reset the mock after `init_client`, since the flush belongs to it rather than to the code under test. Every skills test file now passes both in isolation and in the full suite. Co-Authored-By: Claude Opus 5 --- packages/client/tests/test_skills.py | 3 +++ packages/client/tests/test_skills_fs.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/packages/client/tests/test_skills.py b/packages/client/tests/test_skills.py index 466efbf0..1b8ebd58 100644 --- a/packages/client/tests/test_skills.py +++ b/packages/client/tests/test_skills.py @@ -1959,6 +1959,9 @@ async def test_no_ld_track_calls_from_accessors( store.put(make_raw_skill(key="a")) store.put(make_raw_skill(key="bad", contentHash="0" * 64)) await init_client(options={"skillStore": store}, client=mock_ld_client) + # See the note in test_no_ld_track_calls_from_write_skills: the + # sdk-info flush belongs to init_client, not to the accessors. + mock_ld_client.track.reset_mock() await get_skill("a") await get_skill("bad") diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 7490aab0..2b6cff98 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -2505,6 +2505,10 @@ async def test_no_ld_track_calls_from_write_skills( self, root: Path, mock_ld_client: Any ) -> None: await init_client(client=mock_ld_client) + # init_client flushes $ld:ai:sdk:info on the first client of a process, + # so it may or may not have fired depending on what ran before. This + # test is about write_skills, so start counting from here. + mock_ld_client.track.reset_mock() await write_skills([_skill("a"), _skill("../evil")], root) From d30d4b02f56fd2a21ddc8de7c8aea4265b37b587 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 12:55:51 -0400 Subject: [PATCH 40/43] fix(client): require https and refuse redirects in the skills transport Every FDv2 request carries the server-side SDK key in Authorization. A plain http:// base URI sent it in cleartext, and urllib's default redirect handler copied it onto any 3xx target, so a redirecting proxy or misconfigured private instance would forward the key to an arbitrary host. FDv2SkillStore now refuses a base URI that is not https://, except plain http:// to a loopback host (localhost, 127.0.0.1, ::1) for local test doubles. The requester's opener installs a redirect handler that follows nothing, so any 3xx other than 304 surfaces as a fatal, non-retried failure that says redirects are not followed. The poll's 304 path is unchanged and now pinned by a test. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 5 + packages/client/agents.md | 8 + .../src/launchdarkly_ai_server/skills_fdv2.py | 94 +++++++++- packages/client/tests/test_skills_fdv2.py | 172 ++++++++++++++++++ 4 files changed, 276 insertions(+), 3 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 9301d320..54eec2f4 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -534,6 +534,11 @@ objects through the `SkillStore` interface and cannot tell which store produced customer-confidential. A mobile key (`mob-…`) or a client-side environment ID raises from the constructor. +**The SDK key goes only where you pointed it.** `base_uri` must be `https://` (plain `http://` +is refused, except to a loopback host for a local test double), and redirects are never +followed, so a 3xx from a proxy or a misconfigured private instance stops delivery rather than +forwarding the key to whatever host the `Location` header names. + **Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an diff --git a/packages/client/agents.md b/packages/client/agents.md index abff7d50..980d3123 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -256,6 +256,14 @@ shape `skills_core.SkillStore` documents; **nothing above that interface knows i change ever seems to require editing an accessor, verification, or `write_skills`, the adapter boundary is wrong. +**The key travels over TLS only, and only to the base URI.** `_require_https_base_uri` refuses a +plain `http://` base URI in the constructor — the SDK key would go out in cleartext — with a +loopback exemption (`localhost`, `127.0.0.1`, `::1`) because the test suite's fake endpoints +listen there; and the opener is built with `_RefuseRedirects`, because `urllib`'s standard +redirect handler copies `Authorization` onto the redirected request, so any 3xx (304 aside, +which is a poll's not-modified answer) surfaces as an `HTTPError` that `_classify_status` maps +to a fatal, non-retried failure instead of a request carrying the key to the `Location` host. + **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6a964eae..c7db1484 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -183,6 +183,44 @@ def _require_server_side_credential(sdk_key: str) -> None: ) +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +"""The only hosts a plain ``http://`` base URI may name: a local test double.""" + + +def _require_https_base_uri(base_uri: str) -> None: + """ + Refuses a base URI that would send the SDK key in cleartext. + + Every request carries the environment's server-side SDK key in + ``Authorization``, so the transport is ``https://`` only. The one exception + is ``http://`` to a loopback host (``localhost``, ``127.0.0.1``, ``::1``), + which never leaves the machine and is what a local test double listens on. + Raises rather than logs, for the same reason the credential check does: a + store that would leak its key should not exist. + """ + if not isinstance(base_uri, str) or not base_uri.strip(): + raise ValueError( + "FDv2SkillStore requires an https:// base URI; none was given." + ) + parts = urllib.parse.urlsplit(base_uri.strip()) + if parts.scheme == "https" and parts.hostname: + return + if parts.scheme == "http" and parts.hostname in _LOOPBACK_HOSTS: + return + if parts.scheme == "http": + raise ValueError( + f"FDv2SkillStore refuses base_uri {base_uri!r}: a plain http:// URI " + "would send the server-side SDK key in cleartext. Use https:// " + "(the default is https://sdk.launchdarkly.com). Plain http:// is " + "allowed only for a loopback host (localhost, 127.0.0.1, ::1) " + "serving a local test double." + ) + raise ValueError( + f"FDv2SkillStore refuses base_uri {base_uri!r}: expected an https:// URI " + "with a host, such as https://sdk.launchdarkly.com." + ) + + # --------------------------------------------------------------------------- # Diagnostics # --------------------------------------------------------------------------- @@ -869,6 +907,14 @@ def _classify_status(status: int, headers: Any) -> Exception: return _FatalTransportError( f"LaunchDarkly returned HTTP 403. {_FORBIDDEN_ADVICE}" ) + if 300 <= status < 400 and status != 304: + return _FatalTransportError( + f"LaunchDarkly returned HTTP {status}, a redirect. Redirects are not " + "followed, so the SDK key is never forwarded to a host other than the " + "base URI. The SDK-facing FDv2 endpoints do not redirect; check the " + "base URI, and any proxy in between, for the address being redirected " + "to." + ) if status in (400, 405, 406, 414, 501): return _FatalTransportError( f"LaunchDarkly returned HTTP {status}, which retrying will not fix. " @@ -924,6 +970,36 @@ def close(self) -> None: pass +class _RefuseRedirects(urllib.request.HTTPRedirectHandler): + """ + A redirect handler that follows nothing. + + The standard handler copies every request header onto the redirected + request, ``Authorization`` included, so a 3xx from a proxy or a misconfigured + private instance would hand the SDK key to whatever host ``Location`` names. + Declining here makes ``urllib`` surface the 3xx as an ``HTTPError``, which + ``_classify_status`` turns into a fatal, non-retried failure. Same-host + redirects are refused too: the endpoints this module calls do not redirect, + and a 304 is not a redirect and never reaches this handler. + """ + + def redirect_request( + self, + req: Any, + fp: Any, + code: Any, + msg: Any, + headers: Any, + newurl: Any, + ) -> None: + return None + + +def _build_opener() -> urllib.request.OpenerDirector: + """The default opener with its redirect handler replaced by a refusing one.""" + return urllib.request.build_opener(_RefuseRedirects) + + @dataclass(frozen=True) class _PollResult: not_modified: bool @@ -952,8 +1028,9 @@ def __init__( self._sdk_key = sdk_key self._base_uri = base_uri.rstrip("/") self._read_timeout = read_timeout - # Injectable, so an alternative transport can be supplied. - self._opener = opener or urllib.request.build_opener() + # Injectable, so an alternative transport can be supplied. The default + # never follows a redirect; see ``_RefuseRedirects``. + self._opener = opener or _build_opener() self._lock = threading.Lock() # The response of a poll in flight, so ``interrupt`` can reach its # socket from another thread. Polling only: a stream's response is @@ -1016,7 +1093,8 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: self._in_flight = None except urllib.error.HTTPError as exc: if exc.code == 304: - # urllib raises on 304 when no redirect handler swallows it. + # urllib raises on any non-2xx, 304 included; it is a current + # answer here, not a redirect, and is handled before classifying. return _PollResult(not_modified=True, events=[], etag=etag) raise _classify_status(exc.code, exc.headers) from exc except Exception as exc: @@ -1177,6 +1255,12 @@ class FDv2SkillStore: **Server-side only.** A mobile key or a client-side environment ID is refused in the constructor. + **The SDK key goes only where it was pointed.** *base_uri* must be + ``https://`` — plain ``http://`` is refused except to a loopback host, for + local test doubles — and redirects are never followed, so a 3xx from a proxy + or a private instance is a fatal failure rather than a request carrying the + key to whatever host ``Location`` named. + **Delivery is in the background; retrieval is not.** A daemon thread owns the connection and fills memory, and ``get_object`` only ever reads what has already arrived. A process that calls ``get_skill`` immediately after @@ -1210,6 +1294,9 @@ def __init__( _requester: Any = None, ) -> None: """ + *base_uri* must be ``https://``; ``http://`` is accepted only for + ``localhost``, ``127.0.0.1`` or ``::1``. Raises ``ValueError`` otherwise. + *mode* is ``"stream"`` by default. Prefer it: a ``delete-object`` reaches a live stream in seconds. ``"poll"`` exists for environments that cannot hold a long-lived connection, and revocation there is one @@ -1230,6 +1317,7 @@ def __init__( payload resets the count. """ _require_server_side_credential(sdk_key) + _require_https_base_uri(base_uri) if mode not in ("stream", "poll"): raise ValueError(f'mode must be "stream" or "poll", got {mode!r}') if poll_interval <= 0: diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 1277fdf3..02806a39 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -46,6 +46,7 @@ FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, _backoff_delay, + _FatalTransportError, _is_skill_event, _ProtocolReader, _RecoverableTransportError, @@ -198,6 +199,9 @@ def __init__(self) -> None: self._streams: list[list[dict[str, Any]]] = [] self._lock = threading.Lock() self.hold_stream_open = False + # When set, every ``/sdk/stream`` answers 307 to this URL instead of + # streaming, so a test can check that the store refuses to follow it. + self.redirect_stream_to: str | None = None self._release = threading.Event() endpoint = self @@ -257,6 +261,7 @@ def queue_poll( status: int = 200, etag: str | None = None, retry_after: str | None = None, + location: str | None = None, ) -> None: with self._lock: self._polls.append( @@ -265,6 +270,7 @@ def queue_poll( "events": payload_events or [], "etag": etag, "retry_after": retry_after, + "location": location, } ) @@ -285,6 +291,8 @@ def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: handler.send_header("ETag", response["etag"]) if response.get("retry_after"): handler.send_header("Retry-After", response["retry_after"]) + if response.get("location"): + handler.send_header("Location", response["location"]) if status in (200,): body = json.dumps({"events": response["events"]}).encode("utf-8") handler.send_header("Content-Type", "application/json") @@ -298,6 +306,12 @@ def _serve_poll(self, handler: BaseHTTPRequestHandler) -> None: def _serve_stream(self, handler: BaseHTTPRequestHandler) -> None: with self._lock: payload_events = self._streams.pop(0) if self._streams else [] + if self.redirect_stream_to: + handler.send_response(307) + handler.send_header("Location", self.redirect_stream_to) + handler.send_header("Content-Length", "0") + handler.end_headers() + return handler.send_response(200) handler.send_header("Content-Type", "text/event-stream") handler.send_header("Cache-Control", "no-cache") @@ -1855,6 +1869,108 @@ def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> N # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Redirects are refused +# --------------------------------------------------------------------------- + + +@pytest.fixture +def second_endpoint() -> Any: + """A second host, to stand for wherever a ``Location`` header points.""" + server = _FakeFDv2Endpoint() + yield server + server.close() + + +class TestRedirectsAreRefused: + """ + ``urllib``'s standard redirect handler copies every request header onto the + redirected request, ``Authorization`` included. The transport's opener + declines every redirect instead, so a 3xx is a fatal, non-retried failure + and the SDK key never reaches the host ``Location`` names. + """ + + @pytest.mark.parametrize("status", [301, 302, 307, 308]) + def test_a_poll_redirect_is_fatal_and_not_followed( + self, endpoint: Any, second_endpoint: Any, status: int + ) -> None: + target = second_endpoint.base_uri + "/sdk/poll" + endpoint.queue_poll(status=status, location=target) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError) as excinfo: + requester.poll(None, None) + assert str(status) in str(excinfo.value) + assert "not followed" in str(excinfo.value) + assert second_endpoint.requests == [] + + def test_a_stream_redirect_is_fatal_and_not_followed( + self, endpoint: Any, second_endpoint: Any + ) -> None: + endpoint.redirect_stream_to = second_endpoint.base_uri + "/sdk/stream" + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError) as excinfo: + requester.stream(None) + assert "307" in str(excinfo.value) + assert second_endpoint.requests == [] + + def test_a_same_host_redirect_is_refused_too(self, endpoint: Any) -> None: + """The endpoints do not redirect, so there is nothing legitimate to follow.""" + endpoint.queue_poll(status=302, location=endpoint.base_uri + "/sdk/poll") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError): + requester.poll(None, None) + assert len(endpoint.requests) == 1 + + def test_the_sdk_key_never_reaches_the_second_host( + self, endpoint: Any, second_endpoint: Any + ) -> None: + """ + End to end through the store: the redirect stops delivery for good, + with no retry spent on it, and the second host sees no request at all — + so no ``Authorization`` header, since that is what following would + have forwarded. + """ + endpoint.queue_poll(status=301, location=second_endpoint.base_uri + "/sdk/poll") + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert wait_until(lambda: store.failed is not None) + assert store.wait_for_skills(timeout=5) is False + assert store.failed is not None + assert "301" in store.failed + assert "never forwarded" in store.failed + assert store.diagnostics.connection_failures == 0 + assert len(endpoint.requests) == 1 + assert [r["authorization"] for r in second_endpoint.requests] == [] + + def test_a_redirect_in_stream_mode_stops_delivery( + self, endpoint: Any, second_endpoint: Any + ) -> None: + endpoint.redirect_stream_to = second_endpoint.base_uri + "/sdk/stream" + store = FDv2SkillStore( + SDK_KEY, base_uri=endpoint.base_uri, mode="stream", initial_backoff=0.01 + ) + with store: + assert wait_until(lambda: store.failed is not None) + assert store.failed is not None + assert "307" in store.failed + assert second_endpoint.requests == [] + + def test_a_redirect_with_no_location_is_still_fatal(self, endpoint: Any) -> None: + endpoint.queue_poll(status=302) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + with pytest.raises(_FatalTransportError): + requester.poll(None, None) + + def test_a_304_is_not_a_redirect(self, endpoint: Any) -> None: + """The refusal must leave the poll's not-modified path exactly as it was.""" + endpoint.queue_poll(status=304) + requester = _Requester(SDK_KEY, endpoint.base_uri, read_timeout=5.0) + result = requester.poll(None, "etag-1") + assert result.not_modified is True + assert result.etag == "etag-1" + + def _per_object_hashless_errors(caplog: Any) -> list[Any]: """The per-object ERROR, as distinct from the whole-payload summary.""" return [ @@ -2154,6 +2270,62 @@ def test_an_unknown_mode_is_refused(self) -> None: FDv2SkillStore(SDK_KEY, mode="mobile") # type: ignore[arg-type] +class TestBaseUriScheme: + """ + Every request carries the SDK key in ``Authorization``, so the base URI is + ``https://`` only. Plain ``http://`` is allowed to a loopback host and + nowhere else: that is what this suite's own endpoints listen on, and it + never leaves the machine. + """ + + def test_a_plain_http_base_uri_is_refused(self) -> None: + with pytest.raises(ValueError, match="cleartext") as excinfo: + FDv2SkillStore(SDK_KEY, base_uri="http://sdk.launchdarkly.com") + assert "https://" in str(excinfo.value) + + def test_a_plain_http_base_uri_is_refused_even_with_a_requester_injected( + self, + ) -> None: + """The check is on the store, not on the socket it happens to open.""" + with pytest.raises(ValueError, match="cleartext"): + FDv2SkillStore( + SDK_KEY, + base_uri="http://relay.internal:8030", + _requester=_FakeRequester(), + ) + + @pytest.mark.parametrize( + "base_uri", + [ + "http://localhost:8030", + "http://127.0.0.1:8030", + "http://[::1]:8030", + "http://LOCALHOST/", + ], + ) + def test_plain_http_to_a_loopback_host_is_allowed(self, base_uri: str) -> None: + assert FDv2SkillStore(SDK_KEY, base_uri=base_uri) is not None + + def test_a_private_address_is_not_loopback(self) -> None: + """Only the machine itself is exempt; the LAN is not.""" + with pytest.raises(ValueError, match="cleartext"): + FDv2SkillStore(SDK_KEY, base_uri="http://10.0.0.5:8030") + + @pytest.mark.parametrize( + "base_uri", + ["", " ", "sdk.launchdarkly.com", "ftp://sdk.launchdarkly.com", "https://"], + ) + def test_anything_but_an_https_url_with_a_host_is_refused( + self, base_uri: str + ) -> None: + with pytest.raises(ValueError, match="https://"): + FDv2SkillStore(SDK_KEY, base_uri=base_uri) + + def test_https_is_accepted(self) -> None: + assert FDv2SkillStore(SDK_KEY, base_uri="https://sdk.example.com/") is not None + assert FDv2SkillStore(SDK_KEY) is not None + + # --------------------------------------------------------------------------- # The eager re-reconcile, end to end over the transport # --------------------------------------------------------------------------- From 6c08ffe2b2bb40a9dd34f7913f3da128d97a5d92 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 13:29:32 -0400 Subject: [PATCH 41/43] fix(client): bound transport reads in memory before verification sees them Verification caps a skill's content at 10 MiB, but the FDv2 transport read a whole poll body and accumulated a whole SSE event before that cap could run, so a large or hostile response was buffered unbounded in memory. MAX_RESPONSE_BYTES (64 MiB) is a memory backstop for the transport, deliberately far above any legitimate payload and independent of the per-skill content cap. Poll bodies are read in chunks up to the bound; stream lines are read with a size argument and each event's bytes are totalled. Crossing it raises _RecoverableTransportError: nothing from the body or event is applied, the reader abandons the in-flight payload, the failure is recorded in connection_failures/last_error, and delivery retries while the committed set stays served. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 4 + packages/client/agents.md | 9 + .../src/launchdarkly_ai_server/skills_fdv2.py | 91 +++++++++- packages/client/tests/test_skills_fdv2.py | 167 +++++++++++++++++- 4 files changed, 261 insertions(+), 10 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 54eec2f4..848b8d52 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -539,6 +539,10 @@ is refused, except to a loopback host for a local test double), and redirects ar followed, so a 3xx from a proxy or a misconfigured private instance stops delivery rather than forwarding the key to whatever host the `Location` header names. +**Reads are memory-bounded.** No poll body or streamed event is held past `MAX_RESPONSE_BYTES` +(64 MiB, far above any real payload); one that crosses it is dropped without being applied, the +store keeps serving what it last held, and delivery retries on its normal backoff. + **Streaming is the default, and it is what makes revocation fast.** A `delete-object` reaches a live stream in seconds; with `mode="poll"` it arrives within one `poll_interval`. Paired with `watch_skills`, a revoked skill's `SKILL.md` leaves the disk without a restart. During an diff --git a/packages/client/agents.md b/packages/client/agents.md index 980d3123..4be460fb 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -264,6 +264,15 @@ redirect handler copies `Authorization` onto the redirected request, so any 3xx which is a poll's not-modified answer) surfaces as an `HTTPError` that `_classify_status` maps to a fatal, non-retried failure instead of a request carrying the key to the `Location` host. +**Reads are memory-bounded.** `_read_bounded` reads a poll body in chunks, and +`_iter_stream_lines`/`_iter_sse` read each line with a size argument and total each event, all +against `MAX_RESPONSE_BYTES` (64 MiB): crossing it raises `_RecoverableTransportError`, so +nothing from that body or event is applied, the delivery loop abandons the reader's in-flight +payload, records the failure in `connection_failures`/`last_error`, and retries on the usual +backoff while the committed set stays served. The bound is a memory backstop for the transport +and is independent of `skills_core.MAX_SKILL_CONTENT_BYTES`, which caps one skill's content at +verification; do not derive one from the other. + **The skill's version is in the object's `key`. `version` is the payload's.** Each version of a skill is its own object on the wire, identified as `:`: diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index c7db1484..6bc7a4f8 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -99,6 +99,21 @@ """Default ``read_timeout`` in ``"stream"`` mode: the longest gap tolerated between two reads. LaunchDarkly's heartbeats arrive well inside this.""" +MAX_RESPONSE_BYTES = 64 * 1024 * 1024 +"""The most the transport will hold in memory from one response. + +A memory backstop, not a content limit. Verification caps each skill's content +at 10 MiB in ``skills_core``; this bound is on what one poll body, or one +streamed event, may accumulate *before* verification can run against it, and +it is deliberately far above any payload LaunchDarkly legitimately serves. The +two bound different things and are set independently. A body or event that +crosses it is dropped unread as a recoverable transport failure: nothing from +it is committed, the store keeps serving what it last held, and the delivery +loop retries on its usual backoff.""" + +_READ_CHUNK_BYTES = 64 * 1024 +"""How much of a poll body is read per call while checking it against the bound.""" + _EVENT_SERVER_INTENT = "server-intent" _EVENT_PUT_OBJECT = "put-object" _EVENT_DELETE_OBJECT = "delete-object" @@ -1086,7 +1101,7 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: status = getattr(response, "status", None) or response.getcode() if status == 304: return _PollResult(not_modified=True, events=[], etag=etag) - body = response.read() + body = _read_bounded(response, MAX_RESPONSE_BYTES) new_etag = response.headers.get("ETag") or etag finally: with self._lock: @@ -1097,6 +1112,8 @@ def poll(self, basis: str | None, etag: str | None) -> _PollResult: # answer here, not a redirect, and is handled before classifying. return _PollResult(not_modified=True, events=[], etag=etag) raise _classify_status(exc.code, exc.headers) from exc + except _RecoverableTransportError: + raise except Exception as exc: raise _RecoverableTransportError( f"polling request failed: {type(exc).__name__}: {exc}" @@ -1124,6 +1141,29 @@ def stream(self, basis: str | None) -> _StreamConnection: return _StreamConnection(response) +def _read_bounded(response: Any, limit: int) -> bytes: + """ + Reads a whole poll body, holding no more than *limit* bytes of it. + + Read in chunks rather than all at once so a body that is never going to be + accepted is abandoned as soon as it crosses the bound, with at most one + byte over it in memory, instead of being buffered whole and measured after. + """ + chunks: list[bytes] = [] + seen = 0 + while True: + chunk = response.read(min(_READ_CHUNK_BYTES, limit + 1 - seen)) + if not chunk: + return b"".join(chunks) + seen += len(chunk) + if seen > limit: + raise _RecoverableTransportError( + f"polling response exceeded the {limit}-byte transport bound " + f"(at least {seen} bytes received); nothing from it was applied" + ) + chunks.append(chunk) + + def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: """ Unwraps ``{"events": [...]}``. Polling and streaming carry identical event @@ -1147,9 +1187,10 @@ def _decode_poll_body(body: bytes) -> list[tuple[str, Any]]: return events -def _iter_stream_lines(response: Any) -> Any: +def _iter_stream_lines(response: Any, limit: int) -> Any: """ - Yields a streaming body's raw lines, presenting a read failure as retryable. + Yields a streaming body's raw lines, presenting a read failure as retryable + and refusing any single line longer than *limit* bytes. A live stream dies mid-body far more often than it refuses to open: a read timeout on a stream that went quiet, a reset, a truncated chunk. Each of @@ -1157,9 +1198,25 @@ def _iter_stream_lines(response: Any) -> Any: only the transport errors this module defines — anything else it reads as a bug and stops for the process lifetime. Connecting is already wrapped in ``_Requester.stream``; this is the same promise for the body. + + Lines are read with a size argument rather than by iterating the response, + because an unbounded ``readline`` buffers until it finds a newline, and a + line that never ends would be held whole before anything here saw it. """ try: - yield from response + while True: + line = response.readline(limit + 1) + if not line: + return + if len(line) > limit: + raise _RecoverableTransportError( + f"an FDv2 stream line exceeded the {limit}-byte transport " + "bound; the connection was dropped and nothing from the " + "in-flight payload was applied" + ) + yield line + except _RecoverableTransportError: + raise except Exception as exc: raise _RecoverableTransportError( f"reading the FDv2 stream failed: {type(exc).__name__}: {exc}" @@ -1172,11 +1229,18 @@ def _iter_sse(response: Any) -> Any: Minimal on purpose: ``event:``/``data:`` fields, multi-line ``data`` joined with newlines, a blank line dispatching, and ``:`` comments skipped. + + One event may accumulate at most ``MAX_RESPONSE_BYTES`` across its lines. + Past that it is a recoverable transport failure: the generator raises, the + delivery loop drops the connection and retries, and the payload in flight + is abandoned rather than committed. """ + limit = MAX_RESPONSE_BYTES try: name: str | None = None data_lines: list[str] = [] - for raw_line in _iter_stream_lines(response): + event_bytes = 0 + for raw_line in _iter_stream_lines(response, limit): line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n") if line == "": if name is not None: @@ -1192,9 +1256,18 @@ def _iter_sse(response: Any) -> Any: yield name, parsed name = None data_lines = [] + event_bytes = 0 continue if line.startswith(":"): continue + event_bytes += len(raw_line) + if event_bytes > limit: + raise _RecoverableTransportError( + f"an FDv2 stream event exceeded the {limit}-byte transport " + f"bound (at least {event_bytes} bytes received); the " + "connection was dropped and nothing from the in-flight " + "payload was applied" + ) field_name, _, value = line.partition(":") value = value[1:] if value.startswith(" ") else value if field_name == "event": @@ -1274,6 +1347,11 @@ class FDv2SkillStore: ``write_skills(on_unavailable="keep")`` correct. ``diagnostics`` and ``failed`` report the degradation. + **Reads are memory-bounded.** No poll body or streamed event is held past + ``MAX_RESPONSE_BYTES``; one that crosses it is dropped unapplied as a + recoverable failure, the store keeps serving what it last held, and + delivery retries. + **What arrives is untrusted.** Raw wire objects are held verbatim and verified at the accessor boundary, not here. In particular an object with no ``contentHash`` is held and then *withheld*; see @@ -1595,6 +1673,9 @@ def _run(self) -> None: # misleading ``last_error`` on a healthy store. return with self._lock: + # Whatever the dropped connection had transferred so far is + # not a payload; the next connection starts one afresh. + self._reader._abandon_in_flight() self._failures += 1 failures = self._failures answered = self._attempt_answered diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 02806a39..1e920ca0 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -37,6 +37,7 @@ get_skill, get_skill_result, init_client, + skills_fdv2, watch_skills, ) from launchdarkly_ai_server.skills_core import SKILL_OBJECT_KIND @@ -45,9 +46,11 @@ DEFAULT_STREAM_READ_TIMEOUT, FDV2_KEY_DELIMITER, FDV2_OBJECT_KIND, + MAX_RESPONSE_BYTES, _backoff_delay, _FatalTransportError, _is_skill_event, + _iter_sse, _ProtocolReader, _RecoverableTransportError, _Requester, @@ -1334,12 +1337,15 @@ class _DyingResponse: def __init__(self, exc: BaseException) -> None: self._exc = exc - - def __iter__(self) -> Any: + self._lines: list[bytes] = [] for event in full_payload(("put-object", put_skill())): - yield f"event: {event['event']}\n".encode() - yield f"data: {json.dumps(event['data'])}\n".encode() - yield b"\n" + self._lines.append(f"event: {event['event']}\n".encode()) + self._lines.append(f"data: {json.dumps(event['data'])}\n".encode()) + self._lines.append(b"\n") + + def readline(self, size: int = -1) -> bytes: + if self._lines: + return self._lines.pop(0) raise self._exc def close(self) -> None: @@ -1874,6 +1880,157 @@ def test_a_listener_that_raises_does_not_kill_delivery(self, endpoint: Any) -> N # --------------------------------------------------------------------------- +class _LineSource: + """A streaming body served from bytes, with the ``readline`` the parser uses.""" + + def __init__(self, data: bytes) -> None: + self._buf = data + self.closed = False + + def readline(self, size: int = -1) -> bytes: + if size < 0: + size = len(self._buf) + newline = self._buf.find(b"\n", 0, size) + end = size if newline < 0 else newline + 1 + line, self._buf = self._buf[:end], self._buf[end:] + return line + + def close(self) -> None: + self.closed = True + + +class TestTransportMemoryBound: + """ + ``MAX_RESPONSE_BYTES`` bounds what one response may put in memory before + verification's per-skill content cap can see any of it. Disk was already + bounded; this is what bounds memory. The cap is patched small here so the + suite does not have to move 64 MiB to prove it. + """ + + def test_the_bound_is_far_above_any_legitimate_payload(self) -> None: + assert MAX_RESPONSE_BYTES == 64 * 1024 * 1024 + + def test_an_over_cap_poll_body_is_not_applied_and_is_retried( + self, endpoint: Any, monkeypatch: Any + ) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 2048) + endpoint.queue_poll(full_payload(("put-object", put_skill(content="x" * 8192)))) + # A long enough backoff to observe the failure before the retry lands. + with poll_store(endpoint, initial_backoff=0.3, max_backoff=0.3) as store: + assert wait_until(lambda: store.diagnostics.connection_failures == 1) + assert "2048-byte transport bound" in (store.diagnostics.last_error or "") + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + assert store.diagnostics.payloads_transferred == 0 + assert store.diagnostics.skill_objects_received == 0 + assert store.failed is None + # The retry is an ordinary poll; the endpoint answers it 304. + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.wait_for_skills(timeout=5) is True + assert wait_until(lambda: store.diagnostics.connection_failures == 0) + assert "2048-byte transport bound" in (store.diagnostics.last_error or "") + assert all(r["path"] == "/sdk/poll" for r in endpoint.requests) + + def test_a_poll_body_exactly_at_the_cap_is_accepted( + self, endpoint: Any, monkeypatch: Any + ) -> None: + payload = full_payload(("put-object", put_skill())) + body = json.dumps({"events": payload}).encode("utf-8") + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", len(body)) + endpoint.queue_poll(payload) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + + def test_a_poll_body_one_byte_over_the_cap_is_refused( + self, endpoint: Any, monkeypatch: Any + ) -> None: + payload = full_payload(("put-object", put_skill())) + body = json.dumps({"events": payload}).encode("utf-8") + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", len(body) - 1) + endpoint.queue_poll(payload) + with poll_store(endpoint, max_consecutive_failures=0) as store: + assert wait_until(lambda: store.failed is not None) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is None + assert f"{len(body) - 1}-byte transport bound" in store.failed + assert f"at least {len(body)} bytes received" in store.failed + + def test_the_default_cap_leaves_ordinary_payloads_alone( + self, endpoint: Any + ) -> None: + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + with poll_store(endpoint) as store: + assert store.wait_for_skills(timeout=5) is True + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.diagnostics.connection_failures == 0 + assert store.diagnostics.last_error is None + + def test_an_over_cap_stream_event_abandons_the_payload_in_flight( + self, endpoint: Any, monkeypatch: Any + ) -> None: + """ + The first payload commits. The second starts, then carries an event + over the cap: that connection is dropped, the half-received payload is + never committed, and the reconnect finds the committed set intact. + """ + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 2048) + endpoint.queue_stream( + full_payload(("put-object", put_skill())) + + events( + ("server-intent", server_intent("xfer-changes")), + ("put-object", put_skill("oversized", content="x" * 8192)), + ("payload-transferred", transferred("basis-2")), + ) + ) + endpoint.hold_stream_open = True + endpoint.queue_stream(events(("server-intent", server_intent("none")))) + store = FDv2SkillStore( + SDK_KEY, + base_uri=endpoint.base_uri, + mode="stream", + initial_backoff=0.01, + max_backoff=0.02, + ) + try: + store.start() + assert store.wait_for_skills(timeout=5) is True + assert wait_until( + lambda: "transport bound" in (store.diagnostics.last_error or "") + ) + assert wait_until(lambda: len(endpoint.requests) >= 2) + assert store.get_object(SKILL_OBJECT_KIND, "pdf-extraction") is not None + assert store.get_object(SKILL_OBJECT_KIND, "oversized") is None + assert store.diagnostics.payloads_transferred == 1 + assert store.diagnostics.skill_objects_received == 1 + assert store.failed is None + assert endpoint.requests[1]["query"].get("basis") == "basis-1" + finally: + store.close() + + def test_a_stream_line_that_never_ends_is_refused_and_the_body_closed( + self, monkeypatch: Any + ) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 1024) + source = _LineSource(b"data: " + b"x" * 4096) + with pytest.raises(_RecoverableTransportError, match="1024-byte"): + list(_iter_sse(source)) + assert source.closed + + def test_an_event_is_measured_across_its_data_lines(self, monkeypatch: Any) -> None: + monkeypatch.setattr(skills_fdv2, "MAX_RESPONSE_BYTES", 1024) + lines = b"".join(b"data: " + b"x" * 500 + b"\n" for _ in range(3)) + source = _LineSource(b"event: put-object\n" + lines + b"\n") + with pytest.raises(_RecoverableTransportError, match="1024-byte"): + list(_iter_sse(source)) + assert source.closed + + def test_multi_line_data_under_the_cap_still_decodes(self) -> None: + source = _LineSource(b'event: put-object\ndata: {"a":\ndata: 1}\n\n') + assert list(_iter_sse(source)) == [("put-object", {"a": 1})] + assert source.closed + + @pytest.fixture def second_endpoint() -> Any: """A second host, to stand for wherever a ``Location`` header points.""" From 45b549c9fb39ffc530b884ad22053a11f6a26a3f Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 13:44:28 -0400 Subject: [PATCH 42/43] fix(client): pin the decision reads under the skills root to descriptors The managed root and every destructive operation under it were already descriptor-relative, but three reads that decide which branch runs still resolved paths after the pin: the existence probe and compare read in _write_one, the directory check and listing in the orphan sweep, and the existence probe in _prune_one. A root or skill directory swapped after the pin could therefore choose the branch. For the prune that meant the probe could be answered "absent" from the attacker's tree, the unlink skipped, the manifest entry dropped and `removed` reported while the revoked skill stayed on disk. Each of _write_one and _prune_one now pins / relative to the root descriptor before deciding anything and holds it through the action: the probe is a follow_symlinks=False stat on the skill descriptor, the compare read goes through _read_regular_file with dir_fd, and the sweep lists os.listdir(fd). open_directory_nofollow raises the new DirectoryMissing (a ValueError subclass) on ENOENT/ENOTDIR so absence is learned from the pin itself rather than a separate path probe. The path checks in _unsafe_path_reason stay as defense in depth; the lstat floor keeps its path-based code. Tests add root-swap races fired the instant before each decision read and extend the no-absolute-path audit to stat, open and listdir. Docs no longer describe a residual for reads under the root. Co-Authored-By: Claude Fable 5.1 --- packages/client/README.md | 16 +- packages/client/agents.md | 44 ++- .../src/launchdarkly_ai_server/safe_fs.py | 26 +- .../src/launchdarkly_ai_server/skills_fs.py | 283 +++++++++------- packages/client/tests/test_skills_fs.py | 303 +++++++++++++++++- 5 files changed, 527 insertions(+), 145 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 848b8d52..ca834fed 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -455,12 +455,16 @@ a skill from a variation is how revocation works — the next reconcile prunes i **Platform bound: the descriptor-pinned guarantee is POSIX-only.** On POSIX the managed root is opened once per reconcile — `O_RDONLY|O_DIRECTORY|O_NOFOLLOW`, checked with `fstat`, and -held until the call returns — and every destructive step under it runs relative to that -descriptor: the per-skill directory is created and opened relative to the root, the unlink and -the `rmdir` relative to the directory, and the manifest write relative to the root. So a -directory swapped for a symlink *after* its checks cannot redirect a write or a delete: the -descriptor names the inode that was checked, which closes the swap window rather than -narrowing it. Because the root itself is pinned this holds for the root and its ancestors too, +held until the call returns — and every step under it runs relative to that descriptor, the +reads that decide an action as much as the action itself: the per-skill directory is created +and opened relative to the root; the manifest read, the check for an existing `SKILL.md`, the +byte comparison against it and the listing for orphaned temp files are all answered from the +pinned directory; and the unlink and the `rmdir` run relative to the directory, the manifest +write relative to the root. So a directory swapped for a symlink *after* its checks cannot +redirect a write or a delete, and cannot change *which* of them happens either — a prune +cannot be talked into skipping its unlink, and a compare cannot be shown a file from outside +the root: the descriptor names the inode that was checked, which closes the swap window rather +than narrowing it. Because the root itself is pinned this holds for the root and its ancestors too, not only for `/` — but only from the instant the root is opened, which is why the checklist below denies the agent write access to every ancestor. Windows has no `*at()` syscall family, so there `write_skills` falls back to a per-component `lstat` check diff --git a/packages/client/agents.md b/packages/client/agents.md index 4be460fb..5533afb3 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -34,7 +34,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/skills_fdv2.py` | Agent Skills, delivery transport — the FDv2 protocol, the wire-key/`version` translation, the held object set, and `FDv2SkillStore`. Sits **below** the store interface; imports `skills_core` only, and nothing imports it | | `src/launchdarkly_ai_server/skills_watch.py` | Agent Skills, eager re-reconcile — `watch_skills` / `SkillWatcher`, wiring the store's change listener to `write_skills`. Sits **above** `skills_fs` and modifies none of it | | `src/launchdarkly_ai_server/skills_fs.py` | Agent Skills, materialization half — `write_skills`, request resolution, the manifest format and on-disk filenames, per-skill reconcile, and pruning | -| `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | +| `src/launchdarkly_ai_server/safe_fs.py` | Descriptor-pinned filesystem primitives — `atomic_write`, `unlink_file`, `pinned_directory`, `open_directory_nofollow`, `open_or_create_directory`, `SymlinkRefused`, `DirectoryMissing`, and the `*at()` capability probe. Owns the descriptor-vs-path platform split; knows nothing about skills | | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | @@ -445,8 +445,9 @@ Store data is **untrusted input**; the transport is not part of the trust bounda manifest does not list, and it is bounded on every axis: inside `//` only, for a key that passes `_key_rejection_reason`; only names `safe_fs.is_temp_name` recognizes, anchored at both ends and asked of `safe_fs` rather than re-spelled (a copy would drift - from the writer); only regular files, with the type read off the descriptor; unlinked - through the pinned descriptor. It never raises and never aborts a run. + from the writer); only regular files, with the type read off the descriptor; listed off + the pinned descriptor (`os.listdir(fd)`), so the names come from the directory that was + pinned; unlinked through that same descriptor. It never raises and never aborts a run. - **A corrupt manifest fails closed**: unreadable, unparseable, not an object, malformed `entries`, larger than `_MAX_MANIFEST_BYTES`, or a `manifestVersion` outside `1 <= v <= MANIFEST_VERSION` means no overwrites and no prunes, brand-new paths may still @@ -464,9 +465,13 @@ Store data is **untrusted input**; the transport is not part of the trust bounda mode `0644` set explicitly (never inherited from the umask, never executable), write, fsync, `os.replace`, fsync the directory. `os.replace` is the single rename call site and must not be swapped for `os.rename`. -- **Every operation under the root goes through a pinned descriptor, not a path.** See - "Descriptor-pinned filesystem access" below. Re-resolving `/` from its path at - write or unlink time reopens a swap window that the checks above cannot cover. +- **Every operation under the root goes through a pinned descriptor, not a path — the + reads that decide an action included.** See "Descriptor-pinned filesystem access" below. + Re-resolving `/` from its path at write or unlink time reopens a swap window + that the checks above cannot cover; re-resolving it for the existence probe, the compare + read or the orphan listing lets a swap choose the *branch* instead — most seriously, a + prune whose probe is answered "absent" from a swapped directory skips its unlink, drops + the manifest entry, and reports `removed` while the revoked skill stays on disk. - **A key valid to the data model may still be unrepresentable on disk.** The model allows 256 characters; `NAME_MAX` is 255 bytes. Windows additionally reserves 22 MS-DOS device names, none of which can be a directory name there: `con`, `prn`, `aux`, `nul`, @@ -615,13 +620,32 @@ primitives live in `safe_fs.py`, which knows nothing about skills: removal into a delete of an attacker-chosen file. A symlink found where this SDK expects its own file raises `SymlinkRefused` rather than being tidied away: the state on disk is not what the caller believes, and that is the caller's to report. `_prune_one` goes - through it; `rmdir` stays path-based and is safe that way, since it fails `ENOTDIR` on a - symlink and only ever succeeds on an empty directory. + through it; `rmdir` is issued relative to the root descriptor, and is safe at the key + since it fails `ENOTDIR` on a symlink and only ever succeeds on an empty directory. +- `open_directory_nofollow` raises `DirectoryMissing` — a `ValueError` subclass — when + nothing at the path is a directory (`ENOENT`, `ENOTDIR`). That is how the skills side + learns a skill directory is absent *from the pin itself*, rather than from a separate + `exists()` on the path that a swap could answer differently: a prune of a file that is + already gone and a sweep of a directory never created both take that branch. + +The reads that decide an action are pinned the same way, in `skills_fs`. Each of `_write_one` +and `_prune_one` pins `/` relative to the root descriptor *before* it decides +anything and holds the pin through the action: the existence probe is +`os.stat(SKILL_FILENAME, dir_fd=skill_fd, follow_symlinks=False)`, the compare read is +`_read_regular_file(SKILL_FILENAME, dir_fd=skill_fd)`, and the orphan sweep lists +`os.listdir(skill_fd)`. So a swap after the pin cannot choose the branch — it cannot have +a prune skip its unlink and still report `removed`, cannot have a compare read adopt or +refuse over a file outside the root, and cannot feed the sweep names from elsewhere. Where +`dir_fd` is `None` (the `lstat` floor) each of these stays path-based, which is the +documented Windows bound. The manifest read was already pinned; nothing under the root is +read by path any more. Every `lstat`, `realpath` and containment check on the skills side lives in one shared `_unsafe_path_reason`, so the write and prune paths cannot drift apart on what counts as -unsafe. `skills_fs._prune_one` spells its symlink check `os.stat(..., follow_symlinks=False)` -rather than `os.lstat`, matching the name the capability probe advertises. +unsafe. Those checks stay, ahead of the pin, as defense in depth — they are not the boundary +and must not be removed. Symlink probes on the descriptor side are spelled +`os.stat(..., follow_symlinks=False)` rather than `os.lstat`, matching the name the +capability probe advertises. `safe_fs.SUPPORTS_DIR_FD` gates all of it, and the probe is not the obvious one. `os.supports_dir_fd` is populated per underlying syscall, and CPython registers `renameat` diff --git a/packages/client/src/launchdarkly_ai_server/safe_fs.py b/packages/client/src/launchdarkly_ai_server/safe_fs.py index 9d3c11f7..b022aafb 100644 --- a/packages/client/src/launchdarkly_ai_server/safe_fs.py +++ b/packages/client/src/launchdarkly_ai_server/safe_fs.py @@ -62,6 +62,21 @@ """ +class DirectoryMissing(ValueError): + """ + Raised by ``open_directory_nofollow`` when nothing at the path is a + directory: the name is absent (``ENOENT``) or names a non-directory + (``ENOTDIR``). + + A ``ValueError`` subclass so a caller that treats every unpinnable directory + alike keeps its single ``except``; a distinct type so one for which "not + there" is an ordinary outcome — a prune whose file is already gone, a sweep + of a directory that was never created — can tell it from a refusal without + matching on a message or an errno. A symlink is never this: it is refused + as a symlink. + """ + + def _at(directory: Path, dir_fd: int | None) -> str | Path: """ What to name *directory* by, given a descriptor for its parent. @@ -91,22 +106,29 @@ def open_directory_nofollow( would fail every operation rather than fall back. Raises ``ValueError`` when the path will not open, or inspect, as a real - directory. + directory — ``DirectoryMissing`` when that is because nothing is there or + what is there is not a directory, so a caller can treat absence as an + outcome rather than a failure. Both are decided by the open itself (or the + ``lstat`` on the floor), never by a separate existence probe on the path. """ if not SUPPORTS_DIR_FD: try: mode = os.lstat(directory).st_mode + except FileNotFoundError as exc: + raise DirectoryMissing(f"the directory does not exist: {exc}") from exc except OSError as exc: raise ValueError(f"the directory could not be inspected: {exc}") from exc if stat.S_ISLNK(mode): raise ValueError("the directory is a symlink") if not stat.S_ISDIR(mode): - raise ValueError("the path is not a directory") + raise DirectoryMissing("the path is not a directory") return None flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) try: fd = os.open(_at(directory, dir_fd), flags, dir_fd=dir_fd) + except (FileNotFoundError, NotADirectoryError) as exc: + raise DirectoryMissing(f"the directory does not exist: {exc}") from exc except OSError as exc: raise ValueError( f"the directory could not be opened without following links: {exc}" diff --git a/packages/client/src/launchdarkly_ai_server/skills_fs.py b/packages/client/src/launchdarkly_ai_server/skills_fs.py index 6ec85fa5..03aaeeb7 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fs.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fs.py @@ -7,10 +7,14 @@ primitives every destructive step goes through live in ``safe_fs.py``. **The invariants.** The managed root is pinned to a descriptor once per -reconcile and every destructive operation runs relative to it. Destructive -operations only ever touch paths ``/.launchdarkly-skills.json`` records -under a matching key. A corrupt manifest suppresses every destructive action, -and an incomplete retrieval suppresses pruning. Content is re-verified +reconcile, and every operation under it — the destructive ones and the reads +that decide them — runs relative to that descriptor or to a skill directory +pinned relative to it. The existence probe, the compare read and the orphan +listing are pinned before they are consulted, so a directory swapped after the +pin cannot change which branch runs, only what a path check reports. +Destructive operations only ever touch paths ``/.launchdarkly-skills.json`` +records under a matching key. A corrupt manifest suppresses every destructive +action, and an incomplete retrieval suppresses pruning. Content is re-verified immediately before the write. The path checks still run, but as defense in depth rather than as the boundary. @@ -34,6 +38,7 @@ from typing import Any, Literal from .safe_fs import ( + DirectoryMissing, SymlinkRefused, atomic_write, is_temp_name, @@ -649,10 +654,11 @@ def _load_manifest( An absent manifest is not corrupt — that is simply a fresh root. Read relative to the root descriptor, because this file decides which files - the SDK may overwrite and delete: the pin has to cover the decision as well - as the actions. That single open also makes absence ``ENOENT`` on the read - itself, and refuses a symlink or FIFO wearing the manifest's name as - corruption rather than following or waiting on it. + the SDK may overwrite and delete: the pin covers the decision as well as the + actions, here as for every other read under the root. That single open also + makes absence ``ENOENT`` on the read itself, and refuses a symlink or FIFO + wearing the manifest's name as corruption rather than following or waiting + on it. """ fresh: dict[str, Any] = {"manifestVersion": MANIFEST_VERSION, "entries": {}} @@ -839,58 +845,73 @@ def failed(message: str) -> ReconcileAction: # Overwrite only what the manifest records as the SDK's under this key. entry = entries.get(relative) managed = isinstance(entry, dict) and entry.get("key") == key - exists = target.exists() - if exists: - # Hash first, and decide from the bytes. The manifest check below is - # what protects a file the SDK did not write, but on its own it also - # refuses one the SDK wrote and was killed before recording it, wedging - # every later reconcile. Comparing the bytes separates those two cases, - # and only content byte-identical to what LaunchDarkly resolved is - # adopted. This exception must not be widened — see agents.md. - try: - on_disk = _read_regular_file(target, max_bytes=len(encoded)) - except OSError as exc: - if not managed: - # A read that failed proves nothing, and must never become an - # overwrite: it is the comparison below that would authorize one. + # The directory is pinned here, before anything is decided, and the pin is + # held through the write: the existence probe, the compare read and the + # rename all resolve against the same descriptor, so nothing swapped in + # between can change which branch runs or where the write lands. Created + # relative to *root_fd* for the same reason — ``mkdir`` follows a symlink at + # its parent. A directory that does not exist yet is created now rather than + # after the decision; that only ever happens when there is no file to + # compare against, so the write that follows is the one that fills it. + try: + with pinned_directory(skill_dir, create=True, dir_fd=root_fd) as skill_fd: + try: + on_disk = _read_skill_file(skill_dir, skill_fd, max_bytes=len(encoded)) + except OSError as exc: + if not managed: + # A read that failed proves nothing, and must never become + # an overwrite: it is the comparison below that would + # authorize one. + return failed( + f"'{relative}' exists, the manifest does not record it as " + f"managed under key '{key}', and it could not be read to " + f"compare against the resolved content: {exc}; refusing to " + "overwrite a file this SDK may not have written" + ) + return failed(f"'{relative}' could not be read: {exc}") + + if on_disk is None: + action: ReconcileActionKind = "written" + elif hashlib.sha256(on_disk).hexdigest() == content_hash: + # Hash first, and decide from the bytes. The manifest check + # below is what protects a file the SDK did not write, but on + # its own it also refuses one the SDK wrote and was killed + # before recording it, wedging every later reconcile. Comparing + # the bytes separates those two cases, and only content + # byte-identical to what LaunchDarkly resolved is adopted. This + # exception must not be widened — see agents.md. + # + # ``skipped_current`` rather than a new action kind: the bytes + # on disk already are the resolved content, as true for an + # adopted file as for one the SDK wrote. Adoption records a + # manifest entry, so the file becomes prunable later. + _update_entry(entries, relative, skill, content_hash) + record_materialized(key, len(encoded), content_hash, "skipped_current") + return ReconcileAction( + key=key, + action="skipped_current", + version=skill.version, + path=str(target), + ) + elif not managed: return failed( - f"'{relative}' exists, the manifest does not record it as " - f"managed under key '{key}', and it could not be read to " - f"compare against the resolved content: {exc}; refusing to " - "overwrite a file this SDK may not have written" + f"'{relative}' exists but the manifest does not record it as " + f"managed under key '{key}'; refusing to overwrite a file this " + "SDK did not write" ) - return failed(f"'{relative}' could not be read: {exc}") - - if hashlib.sha256(on_disk).hexdigest() == content_hash: - # ``skipped_current`` rather than a new action kind: the bytes on - # disk already are the resolved content, as true for an adopted file - # as for one the SDK wrote. Adoption records a manifest entry, so - # the file becomes prunable later. - _update_entry(entries, relative, skill, content_hash) - record_materialized(key, len(encoded), content_hash, "skipped_current") - return ReconcileAction( - key=key, - action="skipped_current", - version=skill.version, - path=str(target), - ) - - if not managed: - return failed( - f"'{relative}' exists but the manifest does not record it as managed " - f"under key '{key}'; refusing to overwrite a file this SDK did not write" - ) - # Stale version or local tampering — LD-resolved content wins. - action: ReconcileActionKind = "updated" - else: - action = "written" + else: + # Stale version or local tampering — LD-resolved content wins. + action = "updated" - write_error = _write_through_descriptor( - skill_dir, encoded, key, relative, root_fd=root_fd - ) - if write_error is not None: - return failed(write_error) + try: + atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=skill_fd) + except OSError as exc: + return failed(f"'{relative}' could not be written: {exc}") + except OSError as exc: + return failed(f"the directory for skill '{key}' could not be created: {exc}") + except ValueError as exc: + return failed(f"'{relative}' was refused: {exc}") _update_entry(entries, relative, skill, content_hash) record_materialized(key, len(encoded), content_hash, action) @@ -899,6 +920,47 @@ def failed(message: str) -> ReconcileAction: ) +def _skill_file_present(skill_dir: Path, skill_fd: int | None) -> bool: + """ + Whether ``SKILL.md`` is present in the pinned *skill_dir*. + + Probed relative to the descriptor, ``follow_symlinks=False``, so the answer + is about the directory that was pinned and not about wherever its path leads + now. Only ``ENOENT`` means absent: any other failure reports present, so the + step that follows — the read or the unlink — is the one that fails and says + why, rather than a probe deciding silently that there was nothing to do. + Path-based on the ``lstat`` floor, where there is no descriptor. + """ + if skill_fd is None: + return (skill_dir / SKILL_FILENAME).exists() + try: + os.stat(SKILL_FILENAME, dir_fd=skill_fd, follow_symlinks=False) + except FileNotFoundError: + return False + except OSError: + return True + return True + + +def _read_skill_file( + skill_dir: Path, skill_fd: int | None, *, max_bytes: int +) -> bytes | None: + """ + The compare read: the bytes at ``SKILL.md`` in the pinned *skill_dir*, or + ``None`` when there is no file there. + + Both the probe and the read resolve against *skill_fd*, so the bytes that + decide adoption, update or refusal are the ones in the directory that was + pinned. Any other failure propagates as the ``OSError`` it was, for the + caller to report. + """ + if not _skill_file_present(skill_dir, skill_fd): + return None + if skill_fd is None: + return _read_regular_file(skill_dir / SKILL_FILENAME, max_bytes=max_bytes) + return _read_regular_file(SKILL_FILENAME, max_bytes=max_bytes, dir_fd=skill_fd) + + def _read_regular_file( target: Path | str, *, max_bytes: int, dir_fd: int | None = None ) -> bytes: @@ -915,6 +977,11 @@ def _read_regular_file( ``max_bytes`` is required, not optional, so a new call site cannot pull an arbitrary file into memory by omission. The read stops at ``max_bytes + 1``, the extra byte distinguishing "at the cap" from "over it". + + Given a *dir_fd*, *target* is a bare filename resolved inside that + descriptor. Every read under the managed root — the manifest and each + compare read — passes one wherever the platform has descriptors, so the + bytes a decision is made from come from the directory that was pinned. """ flags = ( os.O_RDONLY @@ -952,27 +1019,29 @@ def _sweep_orphan_temp_files(root: Path, root_fd: int | None, key: str) -> None: it is bounded on every axis: inside ``//`` only, for a key that passes ``_key_rejection_reason``; only names ``safe_fs`` recognizes, asked of ``safe_fs`` so the recognizer cannot drift from the writer; only regular - files; every removal descriptor-relative. Never widen these — see agents.md. + files; the listing read off the pinned descriptor, and every removal + relative to it. Never widen these — see agents.md. Never raises and never aborts the run: the reconcile has succeeded either - way, so a sweep that cannot happen is a warning. + way, so a sweep that cannot happen is a warning. A directory that does not + exist is not a failure — there is nothing to sweep, and it is the pin that + says so rather than a separate probe of the path. """ if _key_rejection_reason(key) is not None: return skill_dir = root / key - if not skill_dir.is_dir(): - return try: with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: - # Listing by path is safe even though the removals are - # descriptor-relative: a name reaches the unlink only if it matches - # the anchored temp pattern, and the unlink resolves it inside the - # pinned directory, so a listing redirected between the pin and here - # can at worst name a file that is not in it. - for name in sorted(os.listdir(skill_dir)): + # ``os.listdir`` accepts the descriptor itself on POSIX, so the + # names come from the directory that was pinned; on the lstat floor + # there is no descriptor and the path is all there is. + listed = os.listdir(skill_dir if dir_fd is None else dir_fd) + for name in sorted(listed): if is_temp_name(name, SKILL_FILENAME): _remove_orphan_temp_file(skill_dir, name, dir_fd) + except DirectoryMissing: + return except (OSError, ValueError) as exc: logger.warning( "orphaned temp files under skill '%s' could not be swept: %s", key, exc @@ -1000,36 +1069,6 @@ def _remove_orphan_temp_file(skill_dir: Path, name: str, dir_fd: int | None) -> logger.warning("an orphaned temp file could not be removed: %s", exc) -def _write_through_descriptor( - skill_dir: Path, - encoded: bytes, - key: str, - relative: str, - *, - root_fd: int | None, -) -> str | None: - """ - Performs the write itself. Returns a failure reason, or ``None`` on success. - - Split out of ``_write_one`` because everything above it decides *whether* to - write and this decides nothing: the directory is pinned and every remaining - step is relative to that descriptor, so no check above can be invalidated by - a swap between here and the rename. The ``mkdir`` is relative to *root_fd* - for the same reason — it follows a symlink at its parent. - """ - try: - with pinned_directory(skill_dir, create=True, dir_fd=root_fd) as dir_fd: - try: - atomic_write(skill_dir, SKILL_FILENAME, encoded, dir_fd=dir_fd) - except OSError as exc: - return f"'{relative}' could not be written: {exc}" - except OSError as exc: - return f"the directory for skill '{key}' could not be created: {exc}" - except ValueError as exc: - return f"'{relative}' was refused: {exc}" - return None - - def _update_entry( entries: dict[str, Any], relative: str, skill: Skill, content_hash: str ) -> None: @@ -1141,28 +1180,24 @@ def _prune( return actions -def _unlink_through_descriptor( - skill_dir: Path, relative: str, *, root_fd: int | None +def _unlink_skill_file( + skill_dir: Path, skill_fd: int | None, relative: str ) -> str | None: """ Performs the removal itself. Returns a failure reason, or ``None`` on success. - The mirror of ``_write_through_descriptor``, split out for the same reason. - The directory is pinned before the unlink because unlink never follows a - trailing symlink but does resolve the directory above it, and the pin is - taken relative to *root_fd* because opening ``/`` by path - re-resolves ````. + *skill_fd* is the descriptor ``_prune_one`` already holds for the directory, + the same one the existence probe was answered from — so the file the probe + found is the file this removes. ``unlink`` never follows a trailing symlink + but does resolve the directory above it, which is why it must not be given + a path here. """ try: - with pinned_directory(skill_dir, dir_fd=root_fd) as dir_fd: - try: - unlink_file(skill_dir, SKILL_FILENAME, dir_fd=dir_fd) - except SymlinkRefused: - return f"'{relative}' was not removed: the target file is a symlink" - except OSError as exc: - return f"'{relative}' could not be removed: {exc}" - except ValueError as exc: - return f"'{relative}' was not removed: {exc}" + unlink_file(skill_dir, SKILL_FILENAME, dir_fd=skill_fd) + except SymlinkRefused: + return f"'{relative}' was not removed: the target file is a symlink" + except OSError as exc: + return f"'{relative}' could not be removed: {exc}" return None @@ -1186,12 +1221,26 @@ def _prune_one( # temp file that nothing else on disk records. _sweep_orphan_temp_files(root, root_fd, key) + # Pinned relative to the root before the existence probe, and held through + # the unlink: a directory swapped after the pin cannot make the probe report + # a file that is not there — or, worse, report nothing where the SDK's file + # still is, which would drop the manifest entry and leave a revoked skill on + # disk with a report that says it was removed. A directory that is not there + # at all is the ordinary case for a file that is already gone. removed_from_disk = False - if target.exists(): - failure = _unlink_through_descriptor(skill_dir, relative, root_fd=root_fd) - if failure is not None: - return _prune_error(key, failure, version) - removed_from_disk = True + try: + with pinned_directory(skill_dir, dir_fd=root_fd) as skill_fd: + if _skill_file_present(skill_dir, skill_fd): + failure = _unlink_skill_file(skill_dir, skill_fd, relative) + if failure is not None: + return _prune_error(key, failure, version) + removed_from_disk = True + except DirectoryMissing: + pass + except ValueError as exc: + return _prune_error(key, f"'{relative}' was not removed: {exc}", version) + + if removed_from_disk: try: # Relative to the root descriptor: rmdir is safe at the key (it # fails ENOTDIR on a symlink and needs an empty directory), but a diff --git a/packages/client/tests/test_skills_fs.py b/packages/client/tests/test_skills_fs.py index 2b6cff98..cd7e7105 100644 --- a/packages/client/tests/test_skills_fs.py +++ b/packages/client/tests/test_skills_fs.py @@ -259,6 +259,95 @@ def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootDuring: return self +class _SwapRootBefore: + """Fires the *root*-swap race the instant before one decision read. + + ``_SwapRootDuring`` intercepts a destructive call. This intercepts a read + that decides whether a destructive call happens at all — the existence + probe before a prune, the compare read before a write, the listing before + the orphan sweep — and swaps the root the moment before it runs, once. Those + reads all come *after* the skill directory is pinned, so a swap here is the + latest possible one: every path check has already passed against the real + root, and the only thing left that could go wrong is the read itself + resolving through the link. + + *trigger* decides which call fires it, and must match both spellings — the + absolute path a path-based read passes and the bare name (or descriptor) a + pinned one passes — for the reason ``_SwapRootDuring`` gives: a trigger + that matched only the path would stop firing the moment the read was + pinned, and these tests would pass while asserting nothing. + """ + + def __init__( + self, + attribute: str, + root: Path, + outside: Path, + trigger: Any, + ) -> None: + self.attribute = attribute + self.root = Path(os.path.realpath(root)) + self.moved_to = self.root.parent / f"{self.root.name}.real" + self.outside = outside + self.trigger = trigger + self.swapped = False + self._real = getattr(os, attribute) + + def __call__(self, first: Any, *args: Any, **kwargs: Any) -> Any: + if not self.swapped and self.trigger(first, kwargs): + os.rename(self.root, self.moved_to) + os.symlink(self.outside, self.root, target_is_directory=True) + self.swapped = True + return self._real(first, *args, **kwargs) + + def install(self, monkeypatch: pytest.MonkeyPatch) -> _SwapRootBefore: + monkeypatch.setattr(skills_fs_module.os, self.attribute, self) + return self + + +def _names_the_skill_file(skill_dir: Path) -> Any: + """A ``_SwapRootBefore`` trigger for a ``stat`` or ``open`` of ``SKILL.md``. + + The bare name is only accepted with a ``dir_fd``: a bare name without one + would resolve against the working directory, which is not a call this SDK + makes, and matching it would fire the swap on some unrelated read. + + The absolute form is only accepted when it follows symlinks. The path + check's ``is_symlink`` also stats the absolute ``SKILL.md``, as an + ``lstat``, and that runs *before* the pin as defense in depth — firing on + it would swap the root under the path checks themselves, which refuse, and + the decision read under test would never be reached. + """ + + def trigger(first: Any, kwargs: dict[str, Any]) -> bool: + if not isinstance(first, (str, os.PathLike)): + return False + named = os.fspath(first) + if named == "SKILL.md": + return kwargs.get("dir_fd") is not None + return ( + Path(named) == skill_dir / "SKILL.md" + and kwargs.get("follow_symlinks", True) is not False + ) + + return trigger + + +def _lists_the_skill_directory(skill_dir: Path) -> Any: + """A ``_SwapRootBefore`` trigger for the orphan sweep's ``listdir``. + + A pinned listing passes the descriptor itself; a path-based one passes + ``/``. + """ + + def trigger(first: Any, kwargs: dict[str, Any]) -> bool: + if isinstance(first, int): + return True + return isinstance(first, (str, os.PathLike)) and Path(first) == skill_dir + + return trigger + + _needs_dir_fd = pytest.mark.skipif( not safe_fs_module.SUPPORTS_DIR_FD, reason="no *at() family on this platform; the per-component lstat floor applies", @@ -1766,9 +1855,10 @@ async def test_a_root_swapped_before_the_manifest_read_cannot_poison_ownership( victim.write_text("the customer's own file\n", encoding="utf-8") outside = tmp_path / "outside" (outside / "victim").mkdir(parents=True) - # Present so the path-based existence probe on the prune path is - # satisfied; without it the run would skip the unlink for a reason that - # has nothing to do with the defense under test. + # Present so that a path-based existence probe, were one to return, + # would be satisfied; without it a run whose probe resolved through the + # swapped root would skip the unlink for a reason that has nothing to + # do with the defense under test. (outside / "victim" / "SKILL.md").write_text("bait\n", encoding="utf-8") _write_manifest( outside, @@ -1794,6 +1884,154 @@ async def test_a_root_swapped_before_the_manifest_read_cannot_poison_ownership( assert [a for a in report.actions if a.action == "removed"] == [] assert victim.read_text(encoding="utf-8") == "the customer's own file\n" + @_needs_dir_fd + async def test_a_root_swapped_before_the_prune_probe_still_removes_the_real_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The existence probe before a prune is answered from the pinned + directory, not from the path. + + Answered from the path, a swap landing between the pin and the probe + made the probe look into the attacker's tree, find nothing, and skip + the unlink — while the prune still dropped the manifest entry and still + reported ``removed``. The revoked skill stayed on disk, now unmanaged, + under a report that said it was gone: the one outcome a revocation must + never have. The attacker's directory is empty here for exactly that + reason — it is the "nothing to remove" answer, planted. + + Pinned, the probe sees the real file, the unlink removes it, and the + report is true. + """ + root = tmp_path / "skills" + root.mkdir() + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + _place_managed(root, "a", SKILL_BODY) + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert [a.action for a in report.actions if a.key == "a"] == ["removed"] + # ``removed`` is true: the real file is gone, and so is its entry. + assert not (race.moved_to / "a" / "SKILL.md").exists() + assert "a/SKILL.md" not in _read_manifest(race.moved_to)["entries"] + # And nothing happened in the attacker's tree. + assert list((outside / "a").iterdir()) == [] + + @_needs_dir_fd + async def test_a_root_swapped_before_the_compare_read_cannot_adopt_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Adoption is decided from the bytes in the pinned directory. + + The real root holds a customer-authored file at the managed path with + no manifest entry; the attacker's tree holds a byte-identical copy of + the resolved content at the same relative path. Read by path, the + compare read landed on the attacker's copy, matched, and adopted — and + the entry it recorded was then written into the *real* manifest through + the held descriptor, claiming the customer's file for the next reconcile + to overwrite or delete. Read through the pin, the bytes are the + customer's, they differ, and the write is refused for the reason the + real root warrants. + """ + root = tmp_path / "skills" + root.mkdir() + target = _place_unmanaged(root, "a", "user authored\n") + outside = tmp_path / "outside" + _place_unmanaged(outside, "a", SKILL_BODY) + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + action = _actions_by_key(report)["a"] + assert action.action == "error" + assert "refusing to overwrite a file this SDK did not write" in ( + action.error or "" + ) + real = race.moved_to / "a" / "SKILL.md" + assert real == Path(str(target).replace(str(root), str(race.moved_to))) + assert real.read_text(encoding="utf-8") == "user authored\n" + assert "a/SKILL.md" not in _read_manifest(race.moved_to)["entries"] + assert (outside / "a" / "SKILL.md").read_text(encoding="utf-8") == SKILL_BODY + + @_needs_dir_fd + async def test_a_root_swapped_before_the_compare_read_cannot_refuse_over_an_outside_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The mirror image: the real root's file *is* the resolved content, + left unrecorded by a reconcile killed before its manifest rewrite, and + the attacker's tree holds something else at the same path. Read by + path, the compare read saw the attacker's bytes, they differed, and the + SDK refused to adopt its own file — wedging that skill on every later + run, on the strength of a file that was never inside the root. Read + through the pin, the file is adopted as ``skipped_current``, which is + what the real root's contents call for. + """ + root = tmp_path / "skills" + root.mkdir() + _place_unmanaged(root, "a", SKILL_BODY) + outside = tmp_path / "outside" + _place_unmanaged(outside, "a", "user authored\n") + race = _SwapRootBefore( + "stat", root, outside, _names_the_skill_file(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert _actions_by_key(report)["a"].action == "skipped_current", ( + _error_messages(report) + ) + assert "a/SKILL.md" in _read_manifest(race.moved_to)["entries"] + assert (race.moved_to / "a" / "SKILL.md").read_text( + encoding="utf-8" + ) == SKILL_BODY + assert (outside / "a" / "SKILL.md").read_text( + encoding="utf-8" + ) == "user authored\n" + assert not (outside / MANIFEST_NAME).exists() + + @_needs_dir_fd + async def test_a_root_swapped_before_the_orphan_listing_cannot_reach_outside( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The orphan sweep lists the pinned directory, not the path. + + The removals were already descriptor-relative, so a listing redirected + into the attacker's tree could never have unlinked anything there; what + it could do is come back with the wrong names — the attacker's, which + the pinned unlink then fails to find — so the real orphan is never + swept and keeps its directory pinned forever. Listed off the descriptor, + the real orphan is found and removed, and the attacker's temp file is + neither listed nor touched. + """ + root = tmp_path / "skills" + root.mkdir() + _place_managed(root, "a", SKILL_BODY) + real_orphan = root / "a" / _temp_name("0123456789abcdef") + real_orphan.write_text("half-written body", encoding="utf-8") + outside = tmp_path / "outside" + (outside / "a").mkdir(parents=True) + planted = outside / "a" / _temp_name("fedcba9876543210") + planted.write_text("bait", encoding="utf-8") + race = _SwapRootBefore( + "listdir", root, outside, _lists_the_skill_directory(root / "a") + ).install(monkeypatch) + + report = await write_skills([_skill("a")], root) + + assert race.swapped is True, "the race never fired; the test proves nothing" + assert report.ok is True, _error_messages(report) + assert not (race.moved_to / "a" / real_orphan.name).exists() + assert planted.read_text(encoding="utf-8") == "bait" + assert sorted(p.name for p in (outside / "a").iterdir()) == [planted.name] + @_needs_dir_fd async def test_every_destructive_call_runs_relative_to_a_descriptor( self, root: Path, monkeypatch: pytest.MonkeyPatch @@ -1812,9 +2050,23 @@ async def test_every_destructive_call_runs_relative_to_a_descriptor( a new call site, not a new attack: one operation added on the full path would reopen the window for that operation alone, and no swap test aimed at the existing sequences would notice. + + The reads that *decide* those calls are held to the same bar. The + existence probe and the compare read name ``SKILL.md``, the manifest + read names the manifest, and each must pass the bare name and a + descriptor; the orphan listing must be handed the descriptor itself. + ``_unsafe_path_reason`` is stubbed out for the run: its checks stat + absolute paths on purpose, as defense in depth ahead of the pin, and + they are not what this audit is about. With them out of the way, every + remaining read that names the skill file or the manifest is a decision + read, and must be pinned. """ + monkeypatch.setattr( + skills_fs_module, "_unsafe_path_reason", lambda *args, **kwargs: None + ) destructive = ("mkdir", "rmdir", "unlink", "replace") - recorded: list[tuple[str, str, bool]] = [] + deciding = ("stat", "open", "listdir") + recorded: list[tuple[str, Any, bool]] = [] def recorder(name: str) -> Any: real = getattr(os, name) @@ -1825,7 +2077,7 @@ def wrapper(first: Any, *args: Any, **kwargs: Any) -> Any: name, os.fspath(first) if isinstance(first, (str, os.PathLike)) - else repr(first), + else first, # replace takes src_dir_fd/dst_dir_fd rather than dir_fd. any("dir_fd" in keyword for keyword in kwargs), ) @@ -1834,7 +2086,7 @@ def wrapper(first: Any, *args: Any, **kwargs: Any) -> Any: return wrapper - for name in destructive: + for name in destructive + deciding: monkeypatch.setattr(safe_fs_module.os, name, recorder(name)) written = await write_skills([_skill("a")], root) @@ -1845,10 +2097,33 @@ def wrapper(first: Any, *args: Any, **kwargs: Any) -> Any: # mkdir, replace (the skill file), replace (the manifest), unlink, # rmdir, replace (the manifest again) — the sequence must have run, or # the audit below is vacuous. - assert {entry[0] for entry in recorded} == set(destructive), recorded + assert {entry[0] for entry in recorded} >= set(destructive), recorded assert [ - entry for entry in recorded if os.path.isabs(entry[1]) or not entry[2] + entry + for entry in recorded + if entry[0] in destructive + and ( + not isinstance(entry[1], str) or os.path.isabs(entry[1]) or not entry[2] + ) + ] == [] + + # The decision reads. The prune run's probe found the file and the + # write run's found none, so both answers were given through a + # descriptor; the sweep listed on both runs. + about_a_file = [ + entry + for entry in recorded + if entry[0] in ("stat", "open") + and isinstance(entry[1], str) + and entry[1].endswith(("SKILL.md", MANIFEST_NAME)) + ] + assert about_a_file != [], recorded + assert [ + entry for entry in about_a_file if os.path.isabs(entry[1]) or not entry[2] ] == [] + listings = [entry for entry in recorded if entry[0] == "listdir"] + assert listings != [], recorded + assert [entry for entry in listings if not isinstance(entry[1], int)] == [] class TestWithoutDirFd: @@ -2746,8 +3021,16 @@ async def test_a_read_failure_on_an_unmanaged_file_refuses( real_open = os.open def refuse_the_target(path: Any, *args: Any, **kwargs: Any) -> int: - if isinstance(path, (str, os.PathLike)) and os.fspath(path) == str(target): - raise PermissionError(13, "Permission denied") + # Both spellings, for the same reason ``_SwapRootDuring`` matches + # both: the compare read names the file as the bare ``SKILL.md`` + # relative to the pinned skill directory wherever the platform has + # descriptors, and as the full path on the lstat floor. + if isinstance(path, (str, os.PathLike)): + named = os.fspath(path) + if named == str(target) or ( + named == "SKILL.md" and kwargs.get("dir_fd") is not None + ): + raise PermissionError(13, "Permission denied") return real_open(path, *args, **kwargs) monkeypatch.setattr(skills_fs_module.os, "open", refuse_the_target) From 5e6313c1dd35ab8c003b121d06152d98751e1057 Mon Sep 17 00:00:00 2001 From: Christie Williams Date: Wed, 16 Sep 2026 14:55:09 -0400 Subject: [PATCH 43/43] fix(client): start a replacement thread when a restart races the give-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``_give_up`` published the terminal reason under the lock but ended delivery after the log line, outside it, leaving the dying thread observable in a half-published state. A ``start`` that landed in that window re-armed the waiters, saw a thread that was alive but no longer delivering, and returned without spawning a replacement — a store reporting no failure with nothing left to deliver, and a waiter that could only time out. Publish the reason and the end of delivery together under the lock, so the only work left outside it is the log line, and have ``start`` read the reason before the re-arm clears it: a thread that has given up is not a delivering one, however alive it looks. Return the retry budget on a re-arm as well. A store that gave up at its failure limit carried the spent count into the restarted run and gave up again on its first recoverable failure, without retrying once. Co-Authored-By: Claude Opus 5 --- .../src/launchdarkly_ai_server/skills_fdv2.py | 23 ++++++- packages/client/tests/test_skills_fdv2.py | 61 +++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py index 6bc7a4f8..c0435d19 100644 --- a/packages/client/src/launchdarkly_ai_server/skills_fdv2.py +++ b/packages/client/src/launchdarkly_ai_server/skills_fdv2.py @@ -1466,8 +1466,17 @@ def start(self) -> FDv2SkillStore: Does not block: use ``wait_for_skills`` when boot ordering matters. """ with self._lock: + # Read before the rearm clears it: a thread inside ``_give_up`` is + # still alive and no longer delivering, so ``is_alive`` on its own + # would have this call adopt a run that is about to return and + # leave a store reporting no failure and never delivering again. + delivering = ( + self._failed_reason is None + and self._thread is not None + and self._thread.is_alive() + ) self._rearm_waiters() - if self._thread is not None and self._thread.is_alive(): + if delivering: # A ``close`` whose join timed out leaves the previous thread # running with the stop flag still set. Clearing it lets that # thread carry on delivering, rather than leaving a store that @@ -1494,6 +1503,10 @@ def _rearm_waiters(self) -> None: """ self._delivery_ended.clear() self._failed_reason = None + # The retry budget belongs to the run that spent it. Carrying it over + # would have a store that gave up after its failure limit give up again + # on the restarted run's first recoverable failure. + self._failures = 0 if not self._first_payload.is_set(): self._released.clear() @@ -1733,14 +1746,18 @@ def _give_up(self, reason: str) -> None: with self._lock: self._failed_reason = reason self._reader.diagnostics.last_error = reason + # Let go of anyone waiting on a first payload that is never coming. + # Recorded beside the reason and under the lock so the two are + # published together: a ``start`` that landed between them would + # re-arm the waiters and then have this dying thread end delivery + # on the fresh run, releasing its waiters before it had answered. + self._end_delivery() logger.error( "Skill delivery has stopped and will not retry: %s. The store keeps " "serving the last content it received; skills will not update until " "the process restarts with a working connection.", reason, ) - # Let go of anyone waiting on a first payload that is never coming. - self._end_delivery() def _apply(self, name: str, data: Any) -> None: """ diff --git a/packages/client/tests/test_skills_fdv2.py b/packages/client/tests/test_skills_fdv2.py index 1e920ca0..d7affd6c 100644 --- a/packages/client/tests/test_skills_fdv2.py +++ b/packages/client/tests/test_skills_fdv2.py @@ -1552,6 +1552,67 @@ def test_a_restarted_store_does_not_report_the_old_failure( assert store.wait_for_skills(timeout=5) is True assert store.failed is None + def test_a_restart_during_the_give_up_still_delivers( + self, endpoint: Any, monkeypatch: Any + ) -> None: + """A restart that races the dying thread must not adopt it. + + ``failed`` becomes readable while the delivery thread is still alive and + winding down. A ``start`` in that window has to spawn a replacement: a + live thread is not by itself a delivering one, and treating it as one + leaves a store reporting no failure with nothing left to deliver. + + The window is held open by blocking the give-up log line, which the + dying thread emits after publishing the reason. + """ + giving_up = threading.Event() + may_finish = threading.Event() + real_error = skills_fdv2.logger.error + + def blocking_error(msg: Any, *args: Any, **kwargs: Any) -> None: + if isinstance(msg, str) and msg.startswith("Skill delivery has stopped"): + giving_up.set() + may_finish.wait(timeout=5) + real_error(msg, *args, **kwargs) + + monkeypatch.setattr(skills_fdv2.logger, "error", blocking_error) + + endpoint.queue_poll(status=401) + with poll_store(endpoint) as store: + assert giving_up.wait(timeout=5) + # The premise of the test: the reason is readable and the thread + # that published it has not returned yet. + assert store.failed is not None + assert store._thread is not None + assert store._thread.is_alive() + + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + may_finish.set() + + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + + def test_a_restart_returns_the_retry_budget(self, endpoint: Any) -> None: + """The retry budget belongs to the run that spent it. + + A store that gave up at its failure limit would otherwise carry the + spent count into the restarted run and give up again on its first + recoverable failure, without retrying once. + """ + store = poll_store(endpoint, max_consecutive_failures=1) + # One over the limit, so the first run retries once and then gives up. + endpoint.queue_poll(status=500) + endpoint.queue_poll(status=500) + with store: + assert wait_until(lambda: store.failed is not None) + + endpoint.queue_poll(status=500) + endpoint.queue_poll(full_payload(("put-object", put_skill()))) + store.start() + assert store.wait_for_skills(timeout=5) is True + assert store.failed is None + def test_a_401_stops_delivery(self, endpoint: Any) -> None: endpoint.queue_poll(status=401) with poll_store(endpoint) as store: