From 3861a77cf3e153392602911995e95a2a473a57aa Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 15 Sep 2026 08:35:57 +0200 Subject: [PATCH 01/22] feat: custom env --- .../environments/adapters/datalayer.py | 54 +- .../environments/adapters/daytona.py | 40 +- code_sandboxes/environments/adapters/e2b.py | 36 +- .../environments/adapters/managed.py | 90 ++- code_sandboxes/environments/adapters/modal.py | 36 +- code_sandboxes/environments/conformance.py | 23 +- code_sandboxes/environments/resolve.py | 13 + code_sandboxes/environments/resolve_conda.py | 765 ++++++++++++++++++ code_sandboxes/environments/spec.py | 121 ++- schemas/environment-v1alpha1.json | 5 +- tests/test_environment_conformance.py | 30 + tests/test_environment_datalayer_builder.py | 39 + tests/test_environment_daytona_builder.py | 32 + tests/test_environment_e2b_builder.py | 40 + tests/test_environment_managed_builders.py | 52 +- tests/test_environment_modal_builder.py | 43 + tests/test_environment_resolve_conda.py | 397 +++++++++ tests/test_environment_spec.py | 27 + 18 files changed, 1736 insertions(+), 107 deletions(-) create mode 100644 code_sandboxes/environments/resolve_conda.py create mode 100644 tests/test_environment_resolve_conda.py diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index f3d82d8..9ead0d6 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -78,6 +78,7 @@ apt_snapshot_in, locked_versions, ) +from ..resolve_conda import conda_lock_protected_pins, is_conda_lock from ..spec import BuildSecret, Environment, command_names_secret __all__ = [ @@ -229,8 +230,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca findings.append( CapabilityFinding( code=CAPABILITY_UNSUPPORTED.code, - message="conda environments are resolved by their own solver, " - "which is not built yet", + message="a conda environment is brought as a `dependencyFile` " + "whose `sourceFormat` is `conda`, not through `packages`", field="spec.packages.python.manager", ) ) @@ -253,7 +254,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca field="spec.platform.architecture", ) ) - if lock_text is not None and not locked_versions(lock_text): + pins_or_lock = lock_text is not None and not is_conda_lock(lock_text) + if pins_or_lock and not locked_versions(lock_text): findings.append( CapabilityFinding( code=SPEC_INVALID.code, @@ -323,18 +325,40 @@ def dockerfile(self, request: BuildRequest) -> str: f"COPY wheelhouse/ {imported_wheelhouse}/", 'RUN pip install --no-cache-dir "uv==0.12.11"', ] - lines.extend( - [ - "COPY lock.txt /opt/datalayer/lock.txt", - # `sync` and not `install`: the artifact holds the lock's set, - # and `--require-hashes` means every byte was the resolved one. - # `--find-links` for what no index has — a protected pin's - # own wheel, the fork's local version above all (E1-04). - "RUN --mount=type=cache,target=/root/.cache/uv " - f"uv pip sync --system --require-hashes --find-links {find_links} " - "/opt/datalayer/lock.txt", - ] - ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): the lock is an `@EXPLICIT` file + # `micromamba create --file` installs without re-solving, and the + # protected pip pins the resolver forced over the pip layer are in + # the lock's own `# datalayer-protected:` header. The conda layer + # goes into the base's own environment; the pip layer follows, so + # the kernel stack (E1-04) is present the same as every source. + pins = conda_lock_protected_pins(request.lock_text) + lines.extend( + [ + "COPY lock.txt /opt/datalayer/lock.txt", + "RUN --mount=type=cache,target=/opt/conda/pkgs " + "micromamba install --yes --name base --file /opt/datalayer/lock.txt", + ] + ) + if pins: + requirements = " ".join(shlex.quote(pin) for pin in pins) + lines.append( + "RUN --mount=type=cache,target=/root/.cache/uv " + f"uv pip install --system --find-links {find_links} {requirements}" + ) + else: + lines.extend( + [ + "COPY lock.txt /opt/datalayer/lock.txt", + # `sync` and not `install`: the artifact holds the lock's set, + # and `--require-hashes` means every byte was the resolved one. + # `--find-links` for what no index has — a protected pin's + # own wheel, the fork's local version above all (E1-04). + "RUN --mount=type=cache,target=/root/.cache/uv " + f"uv pip sync --system --require-hashes --find-links {find_links} " + "/opt/datalayer/lock.txt", + ] + ) # A build secret is mounted on the postInstall commands that name it # and nowhere else (§4.1, D-11): never an `ARG` or `ENV`, which bakes a # value into the image's history, never the package-install or files diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index 0a56c2c..3e5813f 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -87,6 +87,7 @@ from __future__ import annotations +import shlex import tempfile import uuid from collections.abc import Callable @@ -110,6 +111,7 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in +from ..resolve_conda import conda_lock_protected_pins, is_conda_lock from ..spec import GPU_SIZE_CLASSES, Environment from .managed import ManagedBuilder @@ -171,6 +173,10 @@ class Builder(ManagedBuilder): variant = "daytona" item = "E2-04" title = "Daytona" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Daytona runs GPUs, on its own hardware and the owner's account (E2-17). #: This builder does not build one yet: see `_own_findings`. gpu = True @@ -343,28 +349,40 @@ def build(self, request: BuildRequest) -> ArtifactReference: # wheelhouse again would only duplicate what `uv pip sync` # can already reach at `WHEELHOUSE_IMAGE_PATH`. Only the # lock is genuinely per-build. - image = ( - image.add_local_file(str(lock_file), _LOCK_PATH) + image = image.add_local_file(str(lock_file), _LOCK_PATH) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` + # reads the `@EXPLICIT` lock without re-solving, and the + # protected pip pins the resolver forced over the pip + # layer come from the lock's own `# datalayer-protected:` + # header, so the kernel stack (E1-04) is present the same + # as for a pip source. + image = image.run_commands( + f"micromamba install --yes --name base --file {_LOCK_PATH}" + ) + pins = conda_lock_protected_pins(request.lock_text) + if pins: + requirements = " ".join(shlex.quote(pin) for pin in pins) + image = image.run_commands( + "pip install --no-cache-dir " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {requirements}" + ) + else: # `uv` is not installed here: the approved base already # bakes it (E1-05, `resolve.py`'s own `bootstrap_uv` # docstring — "an approved Datalayer base already has it - # baked in"), and this phase's `build_sources` is - # `("packages",)` only, so every build starts from that - # base. Reinstalling it added an extra un-hashed network - # fetch outside the resolved lock for no reason (found in - # review) — matching the Datalayer builder's own - # `dockerfile()`, which installs `uv` only for the - # `image` source, not implemented for this variant yet. + # baked in"). Reinstalling it added an extra un-hashed + # network fetch outside the resolved lock for no reason + # (found in review). # # Packages install as root, the same reason the # Datalayer and E2B builders give: a user install lands # under the content directory's own home, which the # runtime mounts over. - .run_commands( + image = image.run_commands( "uv pip sync --system --require-hashes " f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}" ) - ) image = image.dockerfile_commands([f"USER 1000:100\nWORKDIR {_CONTENT_DIR}"]) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) diff --git a/code_sandboxes/environments/adapters/e2b.py b/code_sandboxes/environments/adapters/e2b.py index ce67135..2297352 100644 --- a/code_sandboxes/environments/adapters/e2b.py +++ b/code_sandboxes/environments/adapters/e2b.py @@ -119,6 +119,7 @@ from __future__ import annotations +import shlex import tempfile from collections.abc import Callable from pathlib import Path @@ -140,6 +141,7 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_PATH, apt_pins_in +from ..resolve_conda import conda_lock_protected_pins, is_conda_lock from ..spec import Environment from .managed import ManagedBuilder @@ -191,6 +193,10 @@ class Builder(ManagedBuilder): variant = "e2b" item = "E2-03" title = "E2B" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Firecracker microVMs: no GPU passthrough. gpu = False #: E0-04's spike found only a registry login for the private base, never @@ -390,15 +396,35 @@ def build(self, request: BuildRequest) -> ArtifactReference: chain.copy("datalayer-sandbox", _DOCTOR_PATH, mode=0o755, user="root") .copy("wheelhouse", _WHEELHOUSE_PATH, user="root") .copy("lock.txt", _LOCK_PATH, user="root") - .run_cmd(f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root") - # Packages install as root (E0-04): a user install lands - # under /home/user, which the runtime mounts over. - .run_cmd( + ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` reads the + # `@EXPLICIT` lock without re-solving, and the protected pip + # pins the resolver forced over the pip layer come from the + # lock's own `# datalayer-protected:` header, so the kernel + # stack (E1-04) is present the same as for a pip source. + chain = chain.run_cmd( + f"micromamba install --yes --name base --file {_LOCK_PATH}", + user="root", + ) + pins = conda_lock_protected_pins(request.lock_text) + if pins: + requirements = " ".join(shlex.quote(pin) for pin in pins) + chain = chain.run_cmd( + f"pip install --no-cache-dir --find-links {_WHEELHOUSE_PATH} " + f"{requirements}", + user="root", + ) + else: + chain = chain.run_cmd( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root" + ).run_cmd( + # Packages install as root (E0-04): a user install lands + # under /home/user, which the runtime mounts over. "uv pip sync --system --require-hashes " f"--find-links {_WHEELHOUSE_PATH} {_LOCK_PATH}", user="root", ) - ) for command in files_step(request.environment, variant=self.variant): chain = chain.run_cmd(command) for command in spec.commands.post_install: diff --git a/code_sandboxes/environments/adapters/managed.py b/code_sandboxes/environments/adapters/managed.py index a236e9c..668c4fc 100644 --- a/code_sandboxes/environments/adapters/managed.py +++ b/code_sandboxes/environments/adapters/managed.py @@ -36,7 +36,7 @@ ValidationResult, ) from ..errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError -from ..spec import GPU_SIZE_CLASSES, Environment +from ..spec import GPU_SIZE_CLASSES, Environment, EnvironmentSpec __all__ = ["ManagedBuilder"] @@ -67,6 +67,11 @@ class ManagedBuilder: supports_build_secrets = True #: The build sources it will accept in this phase. build_sources: tuple[str, ...] = ("packages",) + #: When `dependencyFile` is among `build_sources`, the `sourceFormat`s the + #: variant's own build actually installs. A conda source (E3-02) installs + #: with `micromamba`; a `pyproject`/`requirements` dependencyFile is not + #: built for a managed variant yet (E3-01), so it is not listed here. + dependency_formats: tuple[str, ...] = () package_managers: tuple[str, ...] = ("uv", "pip") #: Dockerfile instructions its own builder does not implement (§6). forbidden_instructions: tuple[str, ...] = () @@ -121,6 +126,8 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: field="spec.build.source", ) ) + elif spec.build.source == "dependencyFile": + findings.extend(self._dependency_file_findings(spec)) if spec.packages.python.manager not in self.package_managers: findings.append( CapabilityFinding( @@ -141,34 +148,7 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) ) if not self.gpu: - # A GPU is asked for two ways, and the spec's own validation - # couples them; a `validate` can be reached before that, so both - # are read here rather than trusting the coupling. - if spec.resources.size_class in GPU_SIZE_CLASSES: - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " - f"built for it. Drop {self.variant} from the variants, or build " - "the GPU classes for modal or daytona, which run them on their " - "own hardware" - ), - field="spec.resources.sizeClass", - ) - ) - elif spec.resources.accelerator != "none": - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so an accelerator cannot be built for it. " - f"Drop {self.variant} from the variants, or build the GPU classes for " - "modal or daytona, which run them on their own hardware" - ), - field="spec.resources.accelerator", - ) - ) + findings.extend(self._gpu_findings(spec)) if self.regions: asked = [ region @@ -201,6 +181,58 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) return findings + def _gpu_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A variant with no GPU refuses either way a GPU is asked for. A GPU + is asked two ways, and the spec's own validation couples them; a + `validate` can be reached before that, so both are read here rather + than trusting the coupling.""" + if spec.resources.size_class in GPU_SIZE_CLASSES: + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " + f"built for it. Drop {self.variant} from the variants, or build " + "the GPU classes for modal or daytona, which run them on their " + "own hardware" + ), + field="spec.resources.sizeClass", + ) + ] + if spec.resources.accelerator != "none": + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so an accelerator cannot be built for it. " + f"Drop {self.variant} from the variants, or build the GPU classes for " + "modal or daytona, which run them on their own hardware" + ), + field="spec.resources.accelerator", + ) + ] + return [] + + def _dependency_file_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A `dependencyFile` this variant accepts still only builds the + `sourceFormat`s it has an install step for (E3-01, E3-02): a conda + file installs with `micromamba`, but a `pyproject` or `requirements` + one is not built for a managed variant yet.""" + source_format = spec.build.dependency_file.source_format + if source_format in self.dependency_formats: + return [] + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"a `{source_format}` dependency file is not built for " + f"{self.title} yet; it builds " + f"{', '.join(self.dependency_formats) or 'no dependency file'}" + ), + field="spec.build.dependencyFile.sourceFormat", + ) + ] + def _own_findings( self, environment: Environment, lock_text: str | None ) -> list[CapabilityFinding]: diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index 391e18f..6872934 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -147,6 +147,7 @@ from ..files import files_step from ..redact import redact from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in +from ..resolve_conda import conda_lock_protected_pins, is_conda_lock from ..spec import GPU_SIZE_CLASSES, BuildSecret, Environment, command_names_secret from .managed import ManagedBuilder @@ -211,6 +212,27 @@ def _scrubbed(text: str, values: dict[str, str]) -> str: return redact(text, values.values()) if values else text +def _install_packages(image: Any, lock_text: str) -> Any: + """The package layer for this lock: a conda source (E3-02) installs the + `@EXPLICIT` lock with Modal's own `micromamba_install` and layers the + protected pip pins the resolver forced (from the lock's own + `# datalayer-protected:` header); a pip source runs `uv pip sync`.""" + if is_conda_lock(lock_text): + image = image.micromamba_install(spec_file=_LOCK_PATH) + pins = conda_lock_protected_pins(lock_text) + if pins: + image = image.pip_install(*pins, find_links=WHEELHOUSE_IMAGE_PATH) + return image + return image.run_commands( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', + # Packages install as root: every Modal build step already runs as + # root regardless of any `USER` line (see the module docstring), so + # this is stating what is already true rather than asking for it. + "uv pip sync --system --require-hashes " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", + ) + + def _post_install( image: Any, commands: list[str], declared: list[BuildSecret], step_secrets: dict[str, Any] ) -> Any: @@ -227,6 +249,10 @@ class Builder(ManagedBuilder): variant = "modal" item = "E2-05" title = "Modal" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba_install`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Modal runs GPUs, in the owner's workspace (E2-17). This builder does #: not build one yet: see `build`'s own guard. gpu = True @@ -403,15 +429,7 @@ def build(self, request: BuildRequest) -> ArtifactReference: image = image.add_local_file( str(entrypoint_file), _ENTRYPOINT_PATH, copy=True ).run_commands(f"chmod +x {_ENTRYPOINT_PATH}") - image = image.run_commands( - f'pip install --no-cache-dir "uv=={_UV_VERSION}"', - # Packages install as root: every Modal build step - # already runs as root regardless of any `USER` line - # (see the module docstring), so this is stating what - # is already true rather than asking for it. - "uv pip sync --system --require-hashes " - f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", - ) + image = _install_packages(image, request.lock_text) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) image = _post_install(image, spec.commands.post_install, declared, step_secrets) diff --git a/code_sandboxes/environments/conformance.py b/code_sandboxes/environments/conformance.py index 200f6d6..0447f68 100644 --- a/code_sandboxes/environments/conformance.py +++ b/code_sandboxes/environments/conformance.py @@ -546,8 +546,13 @@ def _gpu(sandbox: Sandbox, requested: bool, cuda: str | None, timeout: float | N problems.append("no GPU is visible") if cuda and not str(answer.get("cuda") or "").startswith(cuda): problems.append(f"CUDA is {answer.get('cuda')}, not {cuda}") + # A GPU version gates on this (E2-17): a version that asked for an + # accelerator and cannot see it, or sees the wrong CUDA, is not the + # version its spec describes. A version that asked for none never + # reaches here (the trivial pass above), so the extended tier still + # gates nothing for a CPU version. return _result( - 11, not problems, gating=False, detail="; ".join(problems) or None, actual=answer + 11, not problems, gating=True, detail="; ".join(problems) or None, actual=answer ) @@ -625,10 +630,17 @@ def run_extended_tier( concurrent_kernels: int = 4, timeout: float | None = 120.0, ) -> ValidationResult: - """Appendix B checks 10-14: recorded per variant, gating nothing.""" + """Appendix B checks 10-14: recorded per variant. Only check 11 gates, and + only for a version that asked for an accelerator (E2-17) — a GPU version + that cannot see its GPU is not what its spec describes; every other + extended check records without gating.""" checks = [ _guard(10, False, lambda: _egress(sandbox, egress_allowed, egress_blocked, timeout)), - _guard(11, False, lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout)), + _guard( + 11, + accelerator_requested, + lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout), + ), _guard(12, False, lambda: _throughput(sandbox, contract, minimum_mib_per_second, timeout)), _guard(13, False, lambda: _cold_start(cold_start_seconds, cold_start_budget)), _guard(14, False, lambda: _concurrent(sandbox, concurrent_kernels, timeout)), @@ -649,7 +661,10 @@ def run_conformance( extended: Mapping[str, Any] | None = None, timeout: float | None = 120.0, ) -> ValidationResult: - """Both tiers: the core tier decides, the extended tier is recorded beside it.""" + """Both tiers together. The core tier decides; the extended tier is recorded + beside it, save for check 11, which also decides for a version that asked for + an accelerator (E2-17) — a GPU version that cannot see its GPU has not built + what its spec described.""" core = run_core_tier( sandbox, python_version=python_version, diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 811d65c..9e408a1 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -1252,6 +1252,19 @@ def resolve_environment( ) dependency_file = environment.spec.build.dependency_file if source == "dependencyFile" and dependency_file is not None: + if dependency_file.source_format == "conda": + # A conda `environment.yml` resolves through its own micromamba + # solve into an explicit lock (E3-02), not uv's pip compile. + from .resolve_conda import resolve_conda_environment + + return resolve_conda_environment( + environment_yml=dependency_file.content, + python_version=environment.spec.language.version, + resolved_bases=resolved_bases, + credential=credential, + log=say, + resolved_at=resolved_at, + ) if dependency_file.source_format == "pyproject": # Verified, not re-resolved (E3-01): the author's own uv.lock is # the answer, and this only proves it still matches pyproject.toml. diff --git a/code_sandboxes/environments/resolve_conda.py b/code_sandboxes/environments/resolve_conda.py new file mode 100644 index 0000000..362de55 --- /dev/null +++ b/code_sandboxes/environments/resolve_conda.py @@ -0,0 +1,765 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an ``environment.yml`` to one explicit lock (E3-02). + +A ``build.source: dependencyFile`` version whose ``sourceFormat`` is ``conda`` +brings a conda ``environment.yml`` — the form advanced users already have for a +project that pulls, say, ``gdal`` from ``conda-forge`` rather than from a wheel. +It is resolved the way every other source is (PLAN_ENV.md §5, D-9): **once**, +into one lock, and every variant of the version then builds from that one lock, +so four sandboxes of the same version carry the same packages. + +Two decisions this item makes, both recorded here rather than left implicit: + +- **micromamba, not conda-lock.** The solve runs ``micromamba`` — the same tool + the Datalayer, E2B and Daytona builders install the lock with, and the tool + Modal's ``micromamba_install`` wraps — so the interpreter that resolves is the + one that installs, with no second solver's opinion in between. The lock it + produces is a conda **explicit** file (``@EXPLICIT``): one ``package-url#hash`` + line per package, which ``micromamba create --file`` installs without + re-solving. That is the "explicit lock with hashes" this box asks for. +- **The protected pins still apply to the pip layer.** A conda environment needs + the same kernel stack every sandbox needs to connect (``ipykernel`` and its + siblings, ``constraints/sandbox-contract-v1.txt``). They are merged *over* the + ``pip:`` section of the ``environment.yml`` exactly as :func:`merge_requirements` + merges them over a ``packages`` source's dependencies — a pip requirement that + contradicts a protected pin is refused with the range that is supported, and + every pin is forced in whether or not the environment named it, because a + ``--constraint`` alone never installs what nothing else already depends on. + +Where the solve runs is the :class:`CondaResolveRunner`'s business, mirroring +:mod:`code_sandboxes.environments.resolve`: + +- :class:`BuildkitCondaResolveRunner` is D-9's: a BuildKit solve ``FROM`` the + resolved base digest, which gives resolution the builder's isolation and its + egress allowlist. +- :class:`MicromambaResolveRunner` runs ``micromamba`` where it is called, for + ``plane local`` and for tests. + +@module code_sandboxes.environments.resolve_conda +""" + +from __future__ import annotations + +import hashlib +import re +import shlex +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Protocol + +from .errors import ( + CAPABILITY_UNSUPPORTED, + PACKAGE_NOT_FOUND, + PROVIDER_ERROR, + RESOLVE_CONFLICT, + SPEC_INVALID, + EnvironmentsError, +) +from .resolve import ( + PROTECTED_PIN_PREFIX, + WHEELHOUSE_IMAGE_PATH, + WHEELHOUSE_PATH, + MergedRequirements, + ProtectedPin, + merge_requirements, +) + +__all__ = [ + "CONDA_LOCK_FORMAT", + "BuildkitCondaResolveRunner", + "CondaEnvironment", + "CondaResolveOutcome", + "CondaResolveRequest", + "CondaResolveRunner", + "MicromambaResolveRunner", + "conda_lock_document", + "conda_lock_protected_pins", + "explicit_lock_packages", + "is_conda_lock", + "merge_conda_pip", + "parse_conda_environment", + "parse_conda_failure", + "resolve_conda_environment", +] + +#: What a conda lock is, as the version records it: a conda **explicit** file, +#: the ``@EXPLICIT`` form ``micromamba create --file`` installs without solving. +CONDA_LOCK_FORMAT = "conda-explicit" + +#: The platform a Phase-2 artifact is built for (D-9): every variant is +#: ``linux/amd64``, so the solve is for ``linux-64`` in conda's own naming. +CONDA_PLATFORM = "linux-64" + +#: How the protected pip pins are recorded in the lock's header, the same +#: prefix :func:`code_sandboxes.environments.resolve.lock_document` uses, so a +#: reader of either lock finds Datalayer's pins the same way. +_EXPLICIT_MARKER = "@EXPLICIT" +_PIP_SECTION_KEY = "pip" + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +# -- Reading the environment.yml ---------------------------------------------- + + +@dataclass(frozen=True) +class CondaEnvironment: + """An ``environment.yml``, as the resolver reads it.""" + + name: str + channels: tuple[str, ...] + conda_dependencies: tuple[str, ...] + pip_dependencies: tuple[str, ...] + + +def parse_conda_environment(text: str) -> CondaEnvironment: + """One ``environment.yml``'s channels, conda packages and pip packages. + + A conda ``dependencies`` list mixes plain conda specs with a single + ``{"pip": [...]}`` mapping for the pip layer; both are separated here so the + protected pins merge over the pip layer alone (:func:`merge_conda_pip`) and + the conda layer is passed through untouched. Anything that is not a conda + spec or the one pip mapping — a nested list, a bare number — is refused with + its position, because a solve is not the place to discover a malformed file. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError as error: + raise EnvironmentsError( + SPEC_INVALID, + f"the environment.yml is not valid YAML: {error}", + detail={"field": "spec.build.dependencyFile.content"}, + ) from error + if not isinstance(document, Mapping): + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml is empty or not a mapping", + detail={"field": "spec.build.dependencyFile.content"}, + ) + raw_dependencies = document.get("dependencies") + if raw_dependencies is None: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names no `dependencies`", + detail={"field": "spec.build.dependencyFile.content"}, + ) + if not isinstance(raw_dependencies, Sequence) or isinstance(raw_dependencies, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`dependencies` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.dependencies"}, + ) + conda: list[str] = [] + pip: list[str] = [] + seen_pip = False + for index, entry in enumerate(raw_dependencies): + field_name = f"spec.build.dependencyFile.content.dependencies[{index}]" + if isinstance(entry, str): + spec = entry.strip() + if spec: + conda.append(spec) + continue + if isinstance(entry, Mapping) and set(entry) == {_PIP_SECTION_KEY}: + if seen_pip: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names more than one `pip:` section", + detail={"field": field_name}, + ) + seen_pip = True + pip.extend(_pip_requirements(entry[_PIP_SECTION_KEY], field_name)) + continue + raise EnvironmentsError( + SPEC_INVALID, + "a `dependencies` entry in the environment.yml is neither a conda " + "spec nor a single `pip:` section", + detail={"field": field_name}, + ) + channels = _channels(document) + return CondaEnvironment( + name=str(document.get("name") or "environment"), + channels=channels, + conda_dependencies=tuple(conda), + pip_dependencies=tuple(pip), + ) + + +def _pip_requirements(entries: Any, field_name: str) -> list[str]: + """The strings of a ``pip:`` section, or a refusal naming its position.""" + if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "the `pip:` section in the environment.yml is not a list", + detail={"field": f"{field_name}.pip"}, + ) + requirements: list[str] = [] + for pip_index, requirement in enumerate(entries): + if not isinstance(requirement, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `pip:` entry in the environment.yml is not a string", + detail={"field": f"{field_name}.pip[{pip_index}]"}, + ) + spec = requirement.strip() + if spec: + requirements.append(spec) + return requirements + + +def _channels(document: Mapping[str, Any]) -> tuple[str, ...]: + raw = document.get("channels") + if raw is None: + return () + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`channels` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.channels"}, + ) + channels: list[str] = [] + for index, channel in enumerate(raw): + if not isinstance(channel, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry in the environment.yml is not a string", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) + text = channel.strip() + if text: + channels.append(text) + return tuple(channels) + + +def merge_conda_pip( + environment: CondaEnvironment, + pins: Sequence[ProtectedPin] | None = None, +) -> MergedRequirements: + """Datalayer's protected pins merged over the environment's ``pip:`` layer. + + The conda layer is Datalayer's to leave alone — the kernel stack is a pip + concern — but the pip layer is merged exactly as a ``packages`` source's + dependencies are (:func:`merge_requirements`): a pip requirement that + contradicts a protected pin is refused, and every pin is forced in whether + or not the environment named it. The one that has no index — the + jupyter-server fork wheel (E1-04) — is satisfied from the wheelhouse the + same ``--find-links`` reaches in the pip step below. + """ + return merge_requirements(environment.pip_dependencies, pins=pins) + + +def rendered_environment( + environment: CondaEnvironment, + merged: MergedRequirements, + *, + python_version: str, +) -> str: + """The ``environment.yml`` the solve is actually given. + + The user's conda packages and channels, with two things settled: the + interpreter is pinned to the base's ``python`` (D-9, never silently + replaced), and the ``pip:`` section is the merged one — the user's pip + requirements with the protected pins forced over them. + """ + import yaml + + conda_without_python = [ + spec for spec in environment.conda_dependencies if _conda_package_name(spec) != "python" + ] + dependencies: list[Any] = [f"python={python_version}", *conda_without_python] + pip_layer = list(merged.requirements) + if pip_layer: + dependencies.append({_PIP_SECTION_KEY: pip_layer}) + document = { + "name": environment.name, + "channels": list(environment.channels) or ["conda-forge"], + "dependencies": dependencies, + } + return yaml.safe_dump(document, sort_keys=False, default_flow_style=False) + + +def _conda_package_name(spec: str) -> str: + """The package a conda spec names, lowercased: ``python`` from + ``python=3.13``, ``python >=3.11`` or ``python[version='3.13']``.""" + return re.split(r"[\s=<>!~\[]", spec.strip(), maxsplit=1)[0].strip().lower() + + +# -- What a runner is asked, and what it answers ------------------------------ + + +@dataclass(frozen=True) +class CondaResolveRequest: + """One conda solve: what to resolve, for which interpreter, from where.""" + + environment_yml: str + """The rendered ``environment.yml`` — interpreter pinned, pins merged.""" + + python_version: str + platform: str = CONDA_PLATFORM + base_reference: str = "" + """The base the solve runs inside, pinned by digest (D-9).""" + + registry_auth: Mapping[str, str] | None = None + """What the registry needs to be read, when the runner pulls the base.""" + + +@dataclass +class CondaResolveOutcome: + """A conda solve's answer: the explicit lock, verbatim from ``micromamba``.""" + + lock_text: str + + +class CondaResolveRunner(Protocol): + """Where a conda solve runs.""" + + name: str + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: ... + + +# -- Reading micromamba's refusals -------------------------------------------- + +_NOTHING_PROVIDES = re.compile( + r"nothing provides (?:requested )?(?P[A-Za-z0-9][A-Za-z0-9._-]*)" +) +_PACKAGE_NOT_FOUND = re.compile( + r"(?:package|libmamba).*?(?P[A-Za-z0-9][A-Za-z0-9._-]*) is not available" +) + + +def parse_conda_failure(output: str) -> EnvironmentsError: + """``micromamba``'s refusal as one of section 10's codes. + + A package no channel serves, and an unsatisfiable set of packages, are the + version's own to fix and are reported as such. Anything else — the channel + unreachable, ``micromamba`` missing — is ``DL_ENV_PROVIDER_ERROR``, which is + retryable, because nothing about the version is wrong. + """ + text = " ".join(line.strip() for line in output.splitlines() if line.strip()) + detail: dict[str, Any] = {"output": text[:2000]} + + provides = _NOTHING_PROVIDES.search(text) + if provides: + name = provides.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + unavailable = _PACKAGE_NOT_FOUND.search(text) + if unavailable: + name = unavailable.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + lowered = text.lower() + if ( + "could not solve" in lowered + or "unsolvable" in lowered + or "encountered problems while solving" in lowered + or "no solution" in lowered + ): + return EnvironmentsError( + RESOLVE_CONFLICT, + "The conda dependencies cannot be satisfied together: " + text[:400], + detail=detail, + ) + return EnvironmentsError( + PROVIDER_ERROR, + "The conda resolver did not finish: " + text[:400], + detail=detail, + ) + + +# -- Running the solve -------------------------------------------------------- + + +class MicromambaResolveRunner: + """``micromamba`` where this runs: for ``plane local`` and for tests. + + Creates the environment the ``environment.yml`` asks for into a scratch + prefix and exports it as an explicit lock. The prefix is thrown away; only + the lock is kept, which is the one thing every variant then builds from. + """ + + name = "micromamba" + + def __init__(self, micromamba: str | None = None, timeout: float = 900.0) -> None: + # `None` means "find it"; an empty string means "there is none", which + # is how a test says so without hiding micromamba from the process. + self._micromamba = (shutil.which("micromamba") or "") if micromamba is None else micromamba + self._timeout = timeout + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._micromamba: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `micromamba` to resolve with: the local conda resolver needs it on the PATH", + detail={"missing": "micromamba", "runner": self.name}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-") as directory: + root = Path(directory) + spec_file = root / "environment.yml" + spec_file.write_text(request.environment_yml, encoding="utf-8") + prefix = root / "prefix" + create = [ + self._micromamba, + "create", + "--yes", + "--prefix", + str(prefix), + "--platform", + request.platform, + "--file", + str(spec_file), + ] + say(f"Resolving the conda environment for {request.platform}") + try: + created = subprocess.run( # noqa: S603 - the argv is built here + create, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env={**_os_environ(), "PIP_FIND_LINKS": str(WHEELHOUSE_PATH)}, + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if created.returncode != 0: + for line in (created.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(created.stderr or created.stdout or "") + export = subprocess.run( # noqa: S603 - the argv is built here + [self._micromamba, "env", "export", "--explicit", "--prefix", str(prefix)], + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + ) + if export.returncode != 0: + for line in (export.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(export.stderr or export.stdout or "") + return CondaResolveOutcome(lock_text=export.stdout) + + +class BuildkitCondaResolveRunner: + """D-9's conda solve: ``micromamba`` inside the resolved base, under BuildKit. + + The solve gets the builder's isolation, its egress allowlist and the exact + interpreter the artifact will have. It needs ``buildctl`` and a reachable + ``buildkitd``, neither of which exists until the build pool is deployed + (E1-06), so until then it refuses by name rather than resolving elsewhere. + """ + + name = "buildkit-conda" + + def __init__( + self, + buildctl: str | None = None, + address: str | None = None, + tlscert: str | None = None, + tlskey: str | None = None, + tlscacert: str | None = None, + timeout: float = 1200.0, + ) -> None: + self._buildctl = (shutil.which("buildctl") or "") if buildctl is None else buildctl + self._tlscert = tlscert or "" + self._tlskey = tlskey or "" + self._tlscacert = tlscacert or "" + self._address = address or "" + self._timeout = timeout + + def _tls_options(self) -> list[str]: + if not (self._tlscert and self._tlskey and self._tlscacert): + return [] + return [ + f"--tlscert={self._tlscert}", + f"--tlskey={self._tlskey}", + f"--tlscacert={self._tlscacert}", + ] + + def dockerfile(self, request: CondaResolveRequest) -> str: + """The solve, as the frontend reads it: create the env, then export it. + + The ``environment.yml`` is a spec field a user wrote, so its path is the + only thing that reaches the ``RUN`` line — its content is a file copied + into the context, never interpolated into a shell command — and the + wheelhouse is brought along for the one protected pin no index has + (E1-04), reached through ``PIP_FIND_LINKS`` the same way the local + runner reaches it. + """ + return "\n".join( + [ + f"FROM {request.base_reference} AS solve", + "USER root", + "WORKDIR /solve", + "COPY environment.yml ./environment.yml", + "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), + "RUN --mount=type=cache,target=/opt/conda/pkgs " + "micromamba create --yes --prefix /solve/prefix " + f"--platform {shlex.quote(request.platform)} --file environment.yml", + "RUN micromamba env export --explicit --prefix /solve/prefix > /solve/lock.txt", + "FROM scratch", + "COPY --from=solve /solve/lock.txt /lock.txt", + ] + ) + "\n" + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._buildctl: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `buildctl` to resolve with: the build pool is not deployed here", + detail={"missing": "buildctl", "runner": self.name, "item": "E1-06"}, + ) + if "@sha256:" not in request.base_reference: + raise EnvironmentsError( + SPEC_INVALID, + "The base is resolved to a digest before anything is solved", + detail={"base": request.base_reference}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-solve-") as directory: + root = Path(directory) + (root / "environment.yml").write_text(request.environment_yml, encoding="utf-8") + (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8") + out = root / "out" + command = [ + self._buildctl, + *(["--addr", self._address] if self._address else []), + *self._tls_options(), + "build", + "--frontend", + "dockerfile.v0", + "--local", + f"context={root}", + "--local", + f"dockerfile={root}", + "--output", + f"type=local,dest={out}", + ] + say(f"Solving the conda lock in {request.base_reference}") + try: + finished = subprocess.run( # noqa: S603 - the argv is built here + command, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env=self._environment(request), + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + for line in (finished.stderr or "").splitlines(): + say(line) + if finished.returncode != 0: + raise parse_conda_failure(finished.stderr or finished.stdout or "") + lock = (out / "lock.txt").read_text(encoding="utf-8") + return CondaResolveOutcome(lock_text=lock) + + def _environment(self, request: CondaResolveRequest) -> dict[str, str] | None: + auth = dict(request.registry_auth or {}) + if not auth: + return None + return {**_os_environ(), **{str(key): str(value) for key, value in auth.items()}} + + +def _os_environ() -> dict[str, str]: + import os + + return dict(os.environ) + + +# -- The lock ----------------------------------------------------------------- + + +def explicit_lock_packages(lock_text: str) -> list[str]: + """Every package an explicit lock installs, one URL per line. + + An ``@EXPLICIT`` file is comments, the ``@EXPLICIT`` marker, and then one + ``https://…/pkg.conda#hash`` line per package; the URLs are what a build + installs and what this counts. + """ + packages: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or line == _EXPLICIT_MARKER: + continue + packages.append(line) + return packages + + +def is_conda_lock(lock_text: str | None) -> bool: + """Whether a lock is a conda explicit lock, and not the pip one. + + A conda lock carries the ``@EXPLICIT`` marker; a pip lock never does. A + builder reads this to install with ``micromamba`` rather than ``uv pip + sync`` — the one signal that travels with the lock text itself, so a + builder handed only :attr:`BuildRequest.lock_text` still knows which it is. + """ + if not lock_text: + return False + return any(line.strip() == _EXPLICIT_MARKER for line in lock_text.splitlines()) + + +def conda_lock_protected_pins(lock_text: str) -> list[str]: + """The pip requirements a conda lock's header records as Datalayer's pins. + + :func:`conda_lock_document` writes the protected pip pins as + ``# datalayer-protected: `` lines above the ``@EXPLICIT`` body. A + builder installs the conda layer from the body and then this pip layer, so + the kernel stack (E1-04) is present the same way it is for every source. + """ + prefix = PROTECTED_PIN_PREFIX.strip() + pins: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if line.startswith(prefix): + requirement = line[len(prefix) :].strip() + if requirement: + pins.append(requirement) + return pins + + +def conda_lock_document( + outcome: CondaResolveOutcome, + *, + python_version: str, + base_reference: str, + merged: MergedRequirements, + platform: str = CONDA_PLATFORM, + resolved_at: datetime | None = None, +) -> dict[str, Any]: + """The stored conda lock: its text, its digest, and what a reader needs. + + The protected pins are written as comments above the explicit lock, the + same ``# datalayer-protected:`` lines the pip lock carries, so the one + document says the whole of what a build installs — the conda packages by + URL and hash, and the pip pins Datalayer forced over the pip layer — while + staying a file ``micromamba create --file`` reads unchanged. + """ + when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() + header = [ + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", + f"# resolved-at: {when}", + f"# python: {python_version}", + f"# platform: {platform}", + f"# base: {base_reference}", + ] + for constraint in merged.constraints: + header.append(f"{PROTECTED_PIN_PREFIX}{constraint}") + body = outcome.lock_text.lstrip("\n") + if _EXPLICIT_MARKER not in {line.strip() for line in body.splitlines()}: + raise EnvironmentsError( + PROVIDER_ERROR, + "micromamba did not produce an explicit lock (no @EXPLICIT marker)", + detail={"format": CONDA_LOCK_FORMAT}, + ) + text = "\n".join(header) + "\n" + body + if not text.endswith("\n"): + text += "\n" + packages = explicit_lock_packages(text) + return { + "digest": "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(), + "format": CONDA_LOCK_FORMAT, + "content": text, + "python_version": python_version, + "package_count": len(packages), + } + + +def resolve_conda_environment( + *, + environment_yml: str, + python_version: str, + resolved_bases: Mapping[str, str], + platform: str = CONDA_PLATFORM, + credential: Any = None, + log: Callable[[str], None] | None = None, + runner: CondaResolveRunner | None = None, + resolved_at: datetime | None = None, +) -> dict[str, Any]: + """A conda version's lock, from its ``environment.yml`` (E3-02). + + This is the conda seam of :func:`code_sandboxes.environments.resolve.resolve_environment`: + it takes the ``environment.yml`` a ``dependencyFile`` source carries and the + bases already resolved for the wanted variants, and answers the same lock + document every other source answers — ``digest``, ``format``, ``content``, + ``python_version``, ``package_count`` — so the workflow stores it the same way. + + Raises + ------ + EnvironmentsError + Everything a person can act on: a malformed ``environment.yml``, a pip + requirement that contradicts a protected pin, a conflict, or a package + no channel serves. + """ + say = log or (lambda _line: None) + environment = parse_conda_environment(environment_yml) + merged = merge_conda_pip(environment) + for note in merged.notes: + say(note) + rendered = rendered_environment(environment, merged, python_version=python_version) + solving_in = resolved_bases.get("datalayer") or next(iter(resolved_bases.values())) + request = CondaResolveRequest( + environment_yml=rendered, + python_version=python_version, + platform=platform, + base_reference=solving_in, + registry_auth=_registry_auth(credential), + ) + outcome = (runner or BuildkitCondaResolveRunner()).solve(request, say) + document = conda_lock_document( + outcome, + python_version=python_version, + base_reference=solving_in, + merged=merged, + platform=platform, + resolved_at=resolved_at, + ) + say(f"Locked {document['package_count']} conda packages as {document['digest']}") + return {**document, "resolved_bases": dict(resolved_bases)} + + +def _registry_auth(credential: Any) -> Mapping[str, str] | None: + """The credential's registry auth, however it carries it — as + :func:`code_sandboxes.environments.resolve._registry_auth` reads it, kept + in step so both resolvers accept the one credential shape durable mints.""" + if credential is None: + return None + for attribute in ("registry_auth", "environment", "env"): + value = getattr(credential, attribute, None) + if callable(value): + value = value() + if isinstance(value, Mapping): + return {str(key): str(item) for key, item in value.items()} + return None diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index f0d8f3c..f8a804c 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -58,6 +58,7 @@ "SIZE_CLASSES", "SUPPORTED_BUILD_SOURCES", "VARIANTS", + "PUBLIC_PACKAGE_INDEX_HOSTS", "Accelerator", "ArtifactStatus", "Base", @@ -84,6 +85,7 @@ "VersionStatus", "assert_publishable", "command_names_secret", + "index_is_public", "parse_environment", "parse_requirements_txt", "publication_findings", @@ -167,6 +169,31 @@ class Platform(_Model): architecture: Literal["linux/amd64"] = "linux/amd64" +#: The package indexes D-12 counts as public: a version may be published only +#: when every index it resolves from is one of these, since a private index is +#: reached with a credential the public does not hold. Matched on host, so the +#: trailing `/simple` or its absence never decides it. `pypi.org` is the index; +#: `files.pythonhosted.org` is where its wheels are served from. +PUBLIC_PACKAGE_INDEX_HOSTS = frozenset( + {"pypi.org", "files.pythonhosted.org"} +) + + +def _package_index_host(url: str) -> str: + """The host an index URL names, lower-cased and without its port, or `""`.""" + from urllib.parse import urlsplit # noqa: PLC0415 + + try: + return (urlsplit(url).hostname or "").lower() + except ValueError: + return "" + + +def index_is_public(url: str) -> bool: + """Whether an index URL is one D-12 lets a published version resolve from.""" + return _package_index_host(url) in PUBLIC_PACKAGE_INDEX_HOSTS + + class PythonPackages(_Model): manager: Literal["uv", "pip", "conda"] = "uv" dependencies: list[str] = Field(default_factory=list) @@ -242,17 +269,20 @@ class Compatibility(_Model): class DependencyFileSpec(_Model): - """A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01). + """A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda + `environment.yml` (E3-01, E3-02). ``requirements`` resolves the way ``packages`` does — the protected constraints merged in, the same solve. ``pyproject`` does not resolve at all: its own ``uv.lock`` is verified against the current ``pyproject.toml`` and exported, never re-solved, because a lock the - author already made is the whole point of bringing one. + author already made is the whole point of bringing one. ``conda`` resolves + the ``environment.yml`` in its own ``micromamba`` solve into an explicit + lock, with the protected constraints merged over its ``pip:`` layer. """ - source_format: Literal["requirements", "pyproject"] = "requirements" - #: The `requirements.txt` text, or the `pyproject.toml` text. + source_format: Literal["requirements", "pyproject", "conda"] = "requirements" + #: The `requirements.txt`, `pyproject.toml` or conda `environment.yml` text. content: str = "" #: The `uv.lock` text. Required, and only meaningful, for `pyproject`. lock_content: str = "" @@ -409,7 +439,11 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings: list[SpecFinding] = [] if not dependency_file.content.strip(): name = ( - "pyproject.toml" if dependency_file.source_format == "pyproject" else "requirements.txt" + "pyproject.toml" + if dependency_file.source_format == "pyproject" + else "environment.yml" + if dependency_file.source_format == "conda" + else "requirements.txt" ) findings.append(SpecFinding(f"{field}.content", f"is empty; it is the {name} text")) elif dependency_file.source_format == "requirements": @@ -419,6 +453,8 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding(f"{field}.content[{index}]", f"`{requirement}`: {problem}") ) + elif dependency_file.source_format == "conda": + findings.extend(_conda_environment_findings(dependency_file.content)) if len(dependency_file.content.encode("utf-8")) > MAX_DEPENDENCY_FILE_BYTES: findings.append( SpecFinding(f"{field}.content", f"is over {MAX_DEPENDENCY_FILE_BYTES} bytes") @@ -438,12 +474,31 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding( f"{field}.lockContent", - "is only read for a `pyproject` source; a `requirements` source resolves fresh", + "is only read for a `pyproject` source; a `requirements` or `conda` source " + "resolves fresh", ) ) return findings +def _conda_environment_findings(content: str) -> list[SpecFinding]: + """An `environment.yml`'s own shape, before it reaches the conda solver (E3-02). + + The same validate-before-resolve rule every source follows: a malformed + `environment.yml` is the version's to fix, and refusing it here — with the + field the resolver would have named — is cheaper than a solve that fails on + it after taking a worker. + """ + from .resolve_conda import parse_conda_environment + + field = "spec.build.dependencyFile.content" + try: + parse_conda_environment(content) + except EnvironmentsError as error: + return [SpecFinding(error.detail.get("field", field), error.message, error.code)] + return [] + + def _image_findings(image: ImageSourceSpec | None) -> list[SpecFinding]: field = "spec.build.image" if image is None: @@ -810,17 +865,15 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: D-12: *"A promoted version becomes public only by being published, and only when every input is public — public indexes, no ``files``, no - ``buildSecrets``, an approved base."* This function holds only the - ``buildSecrets`` half of that boundary — a build secret is IAM-held and - fetched for one build's own use, so it is never public by definition, - whether the version is otherwise made of nothing but public inputs or - not. The rest of D-12's boundary (public indexes, no baked ``files``, an - approved base) belongs to the publish route itself once it exists - (E2-15, not built yet as of this writing — there is no - ``services/library`` "environment" artifact type and no publish endpoint - in ``services/runtimes/datalayer_runtimes/services/environments.py`` to - call this from today). This is the seam that route calls when it lands, - named the way every other rule of the specification is. + ``buildSecrets``, an approved base."* This function holds the input half + of that boundary that the spec alone decides against a credential the + public does not hold: a build secret is IAM-held and so never public, and + a private package index is reached with a credential no public reader has. + The parts of D-12 that depend on a version's *status* rather than its + spec — a passing scan, a signed artifact, and that the datalayer variant's + base is an approved one — are the publish route's own to check against the + artifact it publishes (E2-15), since ``publication_findings`` is handed + the spec and nothing built from it. Deliberately never applied to `promote()` (the private, per-owner lifecycle step that makes a version an environment's active one): a @@ -828,17 +881,33 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: ever builds or launches it (D-12's own words). Only the act of making a version world-visible is refused. """ - if not environment.spec.build_secrets: - return [] - ids = ", ".join(secret.id for secret in environment.spec.build_secrets) - return [ - SpecFinding( - "spec.buildSecrets", - f"a version with a build secret ({ids}) can never be published to the " - "public Library (D-12); remove it, or keep the version private", - PUBLICATION_BLOCKED, + findings: list[SpecFinding] = [] + if environment.spec.build_secrets: + ids = ", ".join(secret.id for secret in environment.spec.build_secrets) + findings.append( + SpecFinding( + "spec.buildSecrets", + f"a version with a build secret ({ids}) can never be published to the " + "public Library (D-12); remove it, or keep the version private", + PUBLICATION_BLOCKED, + ) ) + private = [ + url + for url in environment.spec.packages.python.indexes + if not index_is_public(url) ] + if private: + findings.append( + SpecFinding( + "spec.packages.python.indexes", + f"a version that resolves from a private index ({', '.join(private)}) " + "can never be published to the public Library (D-12); publish only from " + f"public indexes ({', '.join(sorted(PUBLIC_PACKAGE_INDEX_HOSTS))})", + PUBLICATION_BLOCKED, + ) + ) + return findings def assert_publishable(environment: Environment) -> None: diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index d23b194..c6bdab7 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -154,7 +154,7 @@ }, "DependencyFileSpec": { "additionalProperties": false, - "description": "A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one.", + "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.", "properties": { "content": { "default": "", @@ -170,7 +170,8 @@ "default": "requirements", "enum": [ "requirements", - "pyproject" + "pyproject", + "conda" ], "title": "Sourceformat", "type": "string" diff --git a/tests/test_environment_conformance.py b/tests/test_environment_conformance.py index a0e8c30..cf1e9d2 100644 --- a/tests/test_environment_conformance.py +++ b/tests/test_environment_conformance.py @@ -282,6 +282,36 @@ def test_egress_and_gpu_are_judged_against_what_was_asked() -> None: assert "CUDA is 12.2, not 12.4" in by_id(result, 11).detail +def test_check_eleven_gates_a_gpu_version_that_cannot_see_its_gpu() -> None: + """E2-17: the GPU check is the one extended check that gates, and only + for a version that asked for an accelerator — a GPU version whose GPU is + not visible is not the version its spec describes.""" + sandbox = ScriptedSandbox({11: {"returncode": 0, "gpus": [], "cuda": None}}) + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating + assert not gpu.passed and not result.passed + assert "no GPU is visible" in gpu.detail + + +def test_check_eleven_gates_when_a_gpu_version_sees_its_gpu() -> None: + """The same version, its GPU and CUDA as the spec asked: the gating check + passes, so the extended tier passes.""" + sandbox = ScriptedSandbox() + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating and gpu.passed and result.passed + + +def test_a_cpu_version_never_gates_on_the_gpu_check() -> None: + """No accelerator asked for: check 11 is the trivial recorded pass, and + the extended tier still gates nothing.""" + result = run_extended_tier(ScriptedSandbox()) + gpu = by_id(result, 11) + assert not gpu.gating and gpu.passed + assert not any(item.gating for item in result.checks) + + def test_the_probes_run_for_real_in_a_local_sandbox( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_environment_datalayer_builder.py b/tests/test_environment_datalayer_builder.py index 5b92936..70b8c7b 100644 --- a/tests/test_environment_datalayer_builder.py +++ b/tests/test_environment_datalayer_builder.py @@ -43,6 +43,28 @@ ) LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02): the `@EXPLICIT` marker, one conda package +#: URL, and the protected pip pins the resolver forced over the pip layer. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-protected: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) + +#: The section 4.1 example as a conda `dependencyFile` source. +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's registry credential, as the workflow mints one (D-17).""" @@ -248,6 +270,23 @@ def test_it_installs_from_the_lock_with_hashes(self) -> None: # Never the loose list: that is the whole point of resolving once. assert "geopandas==1.1.1" not in dockerfile + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): the `@EXPLICIT` lock installs with + `micromamba`, and the protected pip pins the resolver forced over the + pip layer follow, so the kernel stack (E1-04) is present the same.""" + request = a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC) + dockerfile = a_builder().dockerfile(request) + assert ( + "micromamba install --yes --name base --file /opt/datalayer/lock.txt" in dockerfile + ) + # The header's own protected pin, installed with pip after the conda layer. + pip = dockerfile.index("uv pip install --system") + micromamba = dockerfile.index("micromamba install") + assert micromamba < pip + assert "ipykernel==7.3.0" in dockerfile + # A conda source never runs the pip-lock `uv pip sync`. + assert "uv pip sync" not in dockerfile + def test_apt_installs_from_the_snapshot_the_lock_pinned_it_at(self) -> None: """A pinned version can leave the live mirror: the build installs from the snapshot the resolver pinned against (D-9).""" diff --git a/tests/test_environment_daytona_builder.py b/tests/test_environment_daytona_builder.py index 3ac05ab..b468296 100644 --- a/tests/test_environment_daytona_builder.py +++ b/tests/test_environment_daytona_builder.py @@ -45,6 +45,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-protected: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" @@ -459,6 +478,19 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync.args[0] assert "--find-links /opt/datalayer/wheelhouse" in sync.args[0] + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): `micromamba install --file` reads the + `@EXPLICIT` lock, and the protected pip pins the resolver forced over + the pip layer follow — never the pip-lock `uv pip sync`.""" + daytona = FakeDaytonaModule() + a_builder(daytona=daytona).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + image = daytona.client.snapshot.create_calls[0].args[0].image + runs = calls_named(image, "run_commands") + micromamba = next(i for i, call in enumerate(runs) if "micromamba install" in call.args[0]) + pip = next(i for i, call in enumerate(runs) if "ipykernel==7.3.0" in call.args[0]) + assert micromamba < pip + assert not any("uv pip sync" in call.args[0] for call in runs) + def test_user_root_brackets_the_install_steps(self) -> None: """Daytona honours the base's `USER`, unlike E2B (E0-04): no synthetic account, just `USER root` around what needs it.""" diff --git a/tests/test_environment_e2b_builder.py b/tests/test_environment_e2b_builder.py index ce93277..9179fc8 100644 --- a/tests/test_environment_e2b_builder.py +++ b/tests/test_environment_e2b_builder.py @@ -35,6 +35,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-protected: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, E2-01).""" @@ -390,6 +409,27 @@ def test_no_apt_step_when_the_lock_pins_none(self) -> None: a_builder(fake).build(a_request()) assert not any(call.name == "run_cmd" and "apt-get" in call.args[0] for call in fake.calls) + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): `micromamba install --file` reads the + `@EXPLICIT` lock, and the protected pip pins the resolver forced over + the pip layer follow — never the pip-lock `uv pip sync`.""" + fake = FakeTemplate() + a_builder(fake).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + micromamba = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micromamba install" in call.args[0] + ) + pip = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "ipykernel==7.3.0" in call.args[0] + ) + assert micromamba < pip + assert not any( + call.name == "run_cmd" and "uv pip sync" in call.args[0] for call in fake.calls + ) + def test_the_owners_credential_is_passed_to_the_sdk(self) -> None: """D-8: a build must run in the *environment owner's* team, never whichever team the worker process itself happens to be configured diff --git a/tests/test_environment_managed_builders.py b/tests/test_environment_managed_builders.py index 977e508..a473928 100644 --- a/tests/test_environment_managed_builders.py +++ b/tests/test_environment_managed_builders.py @@ -52,6 +52,30 @@ def environment(**spec: Any) -> Environment: return parse_environment(data) +#: A `dependencyFile` conda source: an `environment.yml` a managed variant +#: builds with `micromamba` (E3-02). +CONDA_ENVIRONMENT_YML = ( + "name: geo\n" + "channels: [conda-forge]\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal\n" +) + + +def a_conda_environment(**spec: Any) -> Environment: + return environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": CONDA_ENVIRONMENT_YML, + }, + }, + **spec, + ) + + def messages(report: CapabilityReport) -> str: return " | ".join(finding.message for finding in report.findings) @@ -96,9 +120,10 @@ def test_each_one_bounds_how_long_a_build_may_take(self) -> None: seconds = get_builder(variant).capabilities().max_build_seconds assert seconds and 0 < seconds <= 60 * 60, variant - def test_this_phase_builds_a_package_list_and_nothing_else(self) -> None: + def test_this_phase_builds_a_package_list_and_a_conda_file(self) -> None: for variant in MANAGED: - assert get_builder(variant).capabilities().build_sources == ("packages",) + sources = get_builder(variant).capabilities().build_sources + assert sources == ("packages", "dependencyFile"), variant # -- what each one refuses ------------------------------------------------------ @@ -208,14 +233,29 @@ def test_a_source_this_phase_does_not_build_is_refused_by_name(self) -> None: report = get_builder(variant).validate(environment(build={"source": "dockerfile"})) assert report.supported is False, variant assert "`dockerfile` is not built for" in messages(report) - assert "it builds packages" in messages(report) + assert "it builds packages, dependencyFile" in messages(report) - def test_conda_is_not_resolved_for_a_managed_variant_yet(self) -> None: + def test_a_conda_dependency_file_is_buildable_on_every_managed_variant(self) -> None: + for variant in MANAGED: + report = get_builder(variant).validate(a_conda_environment()) + assert report.supported is True, f"{variant}: {messages(report)}" + + def test_a_pyproject_dependency_file_is_not_built_for_a_managed_variant_yet(self) -> None: report = get_builder("e2b").validate( - environment(packages={"python": {"manager": "conda", "dependencies": ["numpy"]}}) + environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "pyproject", + "content": "[project]\nname='x'\nversion='0'\n", + "lockContent": "# lock\n", + }, + } + ) ) assert report.supported is False - assert "`conda` is not resolved for E2B yet" in messages(report) + assert "a `pyproject` dependency file is not built for E2B yet" in messages(report) + assert "spec.build.dependencyFile.sourceFormat" in fields(report) def test_e2b_and_daytona_refuse_a_build_secret_e0_04_found_no_mechanism_for(self) -> None: """E0-04's spike found only a registry login for the private base on diff --git a/tests/test_environment_modal_builder.py b/tests/test_environment_modal_builder.py index ac8e90d..fcf0001 100644 --- a/tests/test_environment_modal_builder.py +++ b/tests/test_environment_modal_builder.py @@ -46,6 +46,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-protected: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" @@ -114,6 +133,14 @@ def run_commands(self, *commands: str, secrets: Any = None) -> FakeImage: self.calls.append(Call("run_commands", commands, kwargs)) return self + def micromamba_install(self, *, spec_file: str) -> FakeImage: + self.calls.append(Call("micromamba_install", (), {"spec_file": spec_file})) + return self + + def pip_install(self, *packages: str, find_links: str | None = None) -> FakeImage: + self.calls.append(Call("pip_install", packages, {"find_links": find_links})) + return self + def workdir(self, path: str) -> FakeImage: self.calls.append(Call("workdir", (path,))) return self @@ -488,6 +515,22 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync assert "--find-links /opt/datalayer/wheelhouse" in sync + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): Modal's own `micromamba_install` reads the + `@EXPLICIT` lock, and `pip_install` layers the protected pip pins the + resolver forced — never the pip-lock `uv pip sync`.""" + modal = FakeModalModule() + a_builder(modal=modal).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + [image] = modal.Image.created + [mamba] = calls_named(image, "micromamba_install") + assert mamba.kwargs["spec_file"] == "/opt/datalayer/lock.txt" + [pip] = calls_named(image, "pip_install") + assert "ipykernel==7.3.0" in pip.args + mamba_at = image.calls.index(mamba) + pip_at = image.calls.index(pip) + assert mamba_at < pip_at + assert not run_commands_containing(image, "uv pip sync") + def test_no_user_line_is_ever_emitted(self) -> None: """Modal ignores `USER` entirely (found live): writing one would be dead code, so this builder never calls `dockerfile_commands` at all.""" diff --git a/tests/test_environment_resolve_conda.py b/tests/test_environment_resolve_conda.py new file mode 100644 index 0000000..1339f22 --- /dev/null +++ b/tests/test_environment_resolve_conda.py @@ -0,0 +1,397 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an environment.yml to one explicit lock (E3-02). + +Like the pip resolver's suite, micromamba's refusals below are **recorded** — +each is what ``micromamba`` actually writes for that input — so the parser is +tested against the solver's own words rather than a paraphrase of them. +""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from code_sandboxes.environments.bases import ApprovedBase +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.resolve import protected_pins +from code_sandboxes.environments.resolve_conda import ( + CONDA_LOCK_FORMAT, + BuildkitCondaResolveRunner, + CondaResolveOutcome, + CondaResolveRequest, + MicromambaResolveRunner, + conda_lock_document, + explicit_lock_packages, + merge_conda_pip, + parse_conda_environment, + parse_conda_failure, + rendered_environment, + resolve_conda_environment, +) +from code_sandboxes.environments.spec import validate_environment + +# -- What micromamba wrote ---------------------------------------------------- + +#: An explicit lock, as ``micromamba env export --explicit`` writes one. +EXPLICIT_LOCK = """\ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#{} +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#{} +""".format("aa" * 32, "bb" * 32) + +#: A package no channel serves. +CONDA_MISSING = """\ +critical libmamba Could not solve for environment specs +The following package could not be found: + - nothing provides no-such-conda-pkg-xyzzy needed by requested +""" + +#: An unsatisfiable set. +CONDA_CONFLICT = """\ +critical libmamba Could not solve for environment specs +The following packages are incompatible +encountered problems while solving: + - package gdal-3.9.2 requires libgdal 3.9.*, but none of the providers can be installed +""" + +#: Not a resolution failure at all — the channel is unreachable. +CONDA_UNREACHABLE = """\ +critical libmamba Download error (6) Could not resolve host: conda.anaconda.org +""" + +BASES = { + "datalayer/python-cpu": ApprovedBase( + ref="datalayer/python-cpu", + python_versions=("3.13",), + channels={ + "2026.09": {"datalayer": "sha256:" + "11" * 32, "modal": "sha256:" + "22" * 32}, + "2026.10": {}, + }, + ) +} + +A_YAML = """\ +name: geospatial +channels: + - conda-forge +dependencies: + - gdal=3.9 + - pip: + - shapely==2.0.6 +""" + + +def a_conda_spec(content: str = A_YAML, **spec: object) -> dict[str, object]: + """A section 4.1 example whose build source is a conda environment.yml.""" + return { + "apiVersion": "environments.datalayer.io/v1alpha1", + "kind": "Environment", + "metadata": {"name": "geospatial", "title": "Geospatial"}, + "spec": { + "language": {"name": "python", "version": "3.13"}, + "base": {"ref": "datalayer/python-cpu", "channel": "2026.09"}, + "build": { + "source": "dependencyFile", + "dependencyFile": {"sourceFormat": "conda", "content": content}, + }, + "resources": {"sizeClass": "medium"}, + "compatibility": {"variants": {"required": ["datalayer"], "optional": ["modal"]}}, + **spec, + }, + } + + +class RecordedRunner: + """A conda solve that answers what it was given, remembering the request.""" + + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome | Exception) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request: CondaResolveRequest, log=None) -> CondaResolveOutcome: + self.request = request + if log is not None: + log("solving conda") + if isinstance(self._outcome, Exception): + raise self._outcome + return self._outcome + + +# -- Reading the environment.yml --------------------------------------------- + + +class TestParsingTheEnvironmentYml: + def test_it_separates_the_conda_and_pip_layers(self) -> None: + env = parse_conda_environment(A_YAML) + assert env.channels == ("conda-forge",) + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == ("shapely==2.0.6",) + + def test_no_pip_section_is_an_empty_pip_layer(self) -> None: + env = parse_conda_environment("dependencies:\n - gdal=3.9\n") + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == () + + def test_malformed_yaml_is_refused_with_its_field(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("dependencies: [\n") + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert caught.value.detail["field"] == "spec.build.dependencyFile.content" + + def test_a_document_that_is_not_a_mapping_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("- just\n- a\n- list\n") + + def test_missing_dependencies_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("name: env\nchannels: [conda-forge]\n") + assert "dependencies" in caught.value.message + + def test_two_pip_sections_are_refused(self) -> None: + text = "dependencies:\n - pip:\n - a\n - pip:\n - b\n" + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert "more than one" in caught.value.message + + def test_a_nested_list_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - [nested]\n") + + def test_a_non_string_pip_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - pip:\n - 3\n") + + def test_channels_that_are_not_a_list_are_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("channels: conda-forge\ndependencies:\n - gdal\n") + + +# -- Datalayer's pins over the pip layer ------------------------------------- + + +class TestMergingThePipLayer: + def test_the_protected_pins_are_forced_into_the_pip_layer(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + names = {req.split("==")[0] for req in merged.requirements} + assert {"shapely", "ipykernel", "jupyter-server"} <= names + + def test_a_pip_requirement_contradicting_a_pin_is_refused(self) -> None: + env = parse_conda_environment( + "dependencies:\n - pip:\n - ipykernel==6.0.0\n" + ) + with pytest.raises(EnvironmentsError) as caught: + merge_conda_pip(env) + assert caught.value.code.code == "DL_ENV_PROTECTED_PACKAGE" + + def test_the_interpreter_is_pinned_and_never_doubled(self) -> None: + env = parse_conda_environment( + "dependencies:\n - python=3.11\n - gdal=3.9\n" + ) + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert rendered.count("python=3.13") == 1 + assert "python=3.11" not in rendered + + def test_the_rendered_pip_layer_carries_the_pins(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert "ipykernel==7.3.0" in rendered + assert "shapely==2.0.6" in rendered + + +# -- Reading micromamba's refusals ------------------------------------------- + + +class TestReadingRefusals: + def test_a_missing_package_is_package_not_found(self) -> None: + error = parse_conda_failure(CONDA_MISSING) + assert error.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + assert "no-such-conda-pkg-xyzzy" in error.message + + def test_an_unsatisfiable_set_is_resolve_conflict(self) -> None: + error = parse_conda_failure(CONDA_CONFLICT) + assert error.code.code == "DL_ENV_RESOLVE_CONFLICT" + + def test_an_unreachable_channel_is_a_provider_error(self) -> None: + error = parse_conda_failure(CONDA_UNREACHABLE) + assert error.code.code == "DL_ENV_PROVIDER_ERROR" + assert error.code.retryable is True + + +# -- The lock ----------------------------------------------------------------- + + +class TestTheLock: + def test_it_counts_only_the_package_urls(self) -> None: + assert explicit_lock_packages(EXPLICIT_LOCK) == [ + "https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#" + "aa" * 32, + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + "bb" * 32, + ] + + def test_the_document_records_the_pins_and_is_deterministic(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + at = datetime(2026, 9, 14, tzinfo=timezone.utc) + document = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + resolved_at=at, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + assert "# datalayer-protected: ipykernel==7.3.0" in document["content"] + assert "@EXPLICIT" in document["content"] + again = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + resolved_at=at, + ) + assert document["digest"] == again["digest"] + + def test_an_export_without_the_marker_is_a_provider_error(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + with pytest.raises(EnvironmentsError) as caught: + conda_lock_document( + CondaResolveOutcome(lock_text="just some lines\nno marker\n"), + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + merged=merged, + ) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + + +# -- The runners refuse honestly when their tool is absent ------------------- + + +class TestTheRunners: + def test_the_local_runner_refuses_without_micromamba(self) -> None: + runner = MicromambaResolveRunner(micromamba="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_the_buildkit_runner_refuses_without_buildctl(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_the_buildkit_runner_refuses_an_unpinned_base(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, python_version="3.13", base_reference="base:latest" + ) + ) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_the_buildkit_dockerfile_brings_the_wheelhouse_and_solves(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + dockerfile = runner.dockerfile( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert "micromamba create" in dockerfile + assert "PIP_FIND_LINKS" in dockerfile + assert "micromamba env export --explicit" in dockerfile + + +# -- The whole resolve, through the recorded runner -------------------------- + + +class TestResolvingACondaVersion: + def test_it_answers_the_lock_and_the_bases(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + document = resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + assert document["resolved_bases"] == {"datalayer": "base@sha256:" + "11" * 32} + + def test_it_sends_the_rendered_environment_with_the_pins(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert runner.request is not None + assert "python=3.13" in runner.request.environment_yml + assert "ipykernel==7.3.0" in runner.request.environment_yml + + def test_a_missing_package_propagates_as_package_not_found(self) -> None: + runner = RecordedRunner(parse_conda_failure(CONDA_MISSING)) + with pytest.raises(EnvironmentsError) as caught: + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert caught.value.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + + +# -- The spec validates a conda dependency file ------------------------------ + + +class TestValidatingTheSpec: + def test_a_conda_environment_file_validates(self) -> None: + # Does not raise: a well-formed conda source is authorable. + validate_environment(a_conda_spec(), bases=BASES) + + def test_an_empty_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content=" \n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_malformed_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content="dependencies: [\n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_lock_content_is_refused_for_conda(self) -> None: + spec = a_conda_spec() + spec["spec"]["build"]["dependencyFile"]["lockContent"] = "irrelevant" # type: ignore[index] + with pytest.raises(EnvironmentsError) as caught: + validate_environment(spec, bases=BASES) + assert "pyproject" in caught.value.message + + +class TestTheProtectedPinsAreTheKernelStack: + def test_merge_uses_the_same_pins_the_pip_resolver_does(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + pin_names = {pin.name for pin in protected_pins()} + forced = {req.split("==")[0].replace("_", "-") for req in merged.requirements} + assert pin_names <= forced diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index 81d03d7..3cb8bc3 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -586,3 +586,30 @@ def test_a_private_build_with_a_secret_is_untouched(self) -> None: accepting the same spec `publication_findings` refuses to publish.""" environment = parse_environment(document()) assert spec_findings(environment) == [] + + def test_a_private_index_blocks_publication(self) -> None: + """D-12: a published version resolves only from public indexes, since a + private one is reached with a credential the public does not hold.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://pypi.mycorp.internal/simple", + ] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.packages.python.indexes" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "pypi.mycorp.internal" in findings[0].message + + def test_only_public_indexes_are_publishable(self) -> None: + """The public index and its wheel host are both accepted; nothing else.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://files.pythonhosted.org/", + ] + environment = parse_environment(data) + assert publication_findings(environment) == [] From c174a07fdb7eb96282b3fc8ecce59b9c5d2e6451 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 15 Sep 2026 10:15:09 +0200 Subject: [PATCH 02/22] env --- .../environments/adapters/datalayer.py | 30 ++- .../environments/adapters/daytona.py | 23 +- code_sandboxes/environments/adapters/e2b.py | 23 +- code_sandboxes/environments/adapters/modal.py | 16 +- code_sandboxes/environments/resolve.py | 13 +- code_sandboxes/environments/resolve_conda.py | 214 +++++++++++++++--- code_sandboxes/environments/spec.py | 78 +++++++ docs/docs/environments/specification.mdx | 4 +- pyproject.toml | 3 +- tests/test_environment_datalayer_builder.py | 18 +- tests/test_environment_daytona_builder.py | 11 +- tests/test_environment_e2b_builder.py | 15 +- tests/test_environment_modal_builder.py | 6 +- tests/test_environment_resolve.py | 48 ++++ tests/test_environment_resolve_conda.py | 81 ++++++- tests/test_environment_spec.py | 29 +++ 16 files changed, 521 insertions(+), 91 deletions(-) diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index 9ead0d6..49b085b 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -78,7 +78,12 @@ apt_snapshot_in, locked_versions, ) -from ..resolve_conda import conda_lock_protected_pins, is_conda_lock +from ..resolve_conda import ( + MICROMAMBA_BINARY, + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_dockerfile_line, +) from ..spec import BuildSecret, Environment, command_names_secret __all__ = [ @@ -327,21 +332,26 @@ def dockerfile(self, request: BuildRequest) -> str: ] if is_conda_lock(request.lock_text): # A conda source (E3-02): the lock is an `@EXPLICIT` file - # `micromamba create --file` installs without re-solving, and the - # protected pip pins the resolver forced over the pip layer are in - # the lock's own `# datalayer-protected:` header. The conda layer - # goes into the base's own environment; the pip layer follows, so - # the kernel stack (E1-04) is present the same as every source. - pins = conda_lock_protected_pins(request.lock_text) + # `micromamba install --file` installs without re-solving, and the + # pip layer the solve resolved — the user's own pip requirements and + # the protected pins forced over them — is in the lock's own + # `# datalayer-pip:` header. The conda layer goes into the base's + # own environment; the pip layer follows, so the kernel stack + # (E1-04) and everything the solve installed is present the same as + # every source. micromamba is copied in from its pinned image + # first: the approved base bakes uv and the wheelhouse but not it. + pip_requirements = conda_lock_pip_requirements(request.lock_text) lines.extend( [ + micromamba_bootstrap_dockerfile_line(), "COPY lock.txt /opt/datalayer/lock.txt", "RUN --mount=type=cache,target=/opt/conda/pkgs " - "micromamba install --yes --name base --file /opt/datalayer/lock.txt", + f"{MICROMAMBA_BINARY} install --yes --name base " + "--file /opt/datalayer/lock.txt", ] ) - if pins: - requirements = " ".join(shlex.quote(pin) for pin in pins) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) lines.append( "RUN --mount=type=cache,target=/root/.cache/uv " f"uv pip install --system --find-links {find_links} {requirements}" diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index 3e5813f..f43dc1a 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -111,7 +111,11 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in -from ..resolve_conda import conda_lock_protected_pins, is_conda_lock +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, +) from ..spec import GPU_SIZE_CLASSES, Environment from .managed import ManagedBuilder @@ -353,16 +357,19 @@ def build(self, request: BuildRequest) -> ArtifactReference: if is_conda_lock(request.lock_text): # A conda source (E3-02): `micromamba install --file` # reads the `@EXPLICIT` lock without re-solving, and the - # protected pip pins the resolver forced over the pip - # layer come from the lock's own `# datalayer-protected:` - # header, so the kernel stack (E1-04) is present the same - # as for a pip source. + # pip layer the solve resolved — the user's pip + # requirements and the protected pins over them — comes + # from the lock's own `# datalayer-pip:` header, so the + # kernel stack (E1-04) and everything the solve installed is + # present the same as for a pip source. micromamba is + # installed first: the approved base bakes uv but not it. + image = image.run_commands(micromamba_bootstrap_command()) image = image.run_commands( f"micromamba install --yes --name base --file {_LOCK_PATH}" ) - pins = conda_lock_protected_pins(request.lock_text) - if pins: - requirements = " ".join(shlex.quote(pin) for pin in pins) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) image = image.run_commands( "pip install --no-cache-dir " f"--find-links {WHEELHOUSE_IMAGE_PATH} {requirements}" diff --git a/code_sandboxes/environments/adapters/e2b.py b/code_sandboxes/environments/adapters/e2b.py index 2297352..875eca2 100644 --- a/code_sandboxes/environments/adapters/e2b.py +++ b/code_sandboxes/environments/adapters/e2b.py @@ -141,7 +141,11 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_PATH, apt_pins_in -from ..resolve_conda import conda_lock_protected_pins, is_conda_lock +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, +) from ..spec import Environment from .managed import ManagedBuilder @@ -399,17 +403,20 @@ def build(self, request: BuildRequest) -> ArtifactReference: ) if is_conda_lock(request.lock_text): # A conda source (E3-02): `micromamba install --file` reads the - # `@EXPLICIT` lock without re-solving, and the protected pip - # pins the resolver forced over the pip layer come from the - # lock's own `# datalayer-protected:` header, so the kernel - # stack (E1-04) is present the same as for a pip source. + # `@EXPLICIT` lock without re-solving, and the pip layer the + # solve resolved — the user's pip requirements and the protected + # pins over them — comes from the lock's own `# datalayer-pip:` + # header, so the kernel stack (E1-04) and everything the solve + # installed is present the same as for a pip source. micromamba + # is installed first: the approved base bakes uv but not it. + chain = chain.run_cmd(micromamba_bootstrap_command(), user="root") chain = chain.run_cmd( f"micromamba install --yes --name base --file {_LOCK_PATH}", user="root", ) - pins = conda_lock_protected_pins(request.lock_text) - if pins: - requirements = " ".join(shlex.quote(pin) for pin in pins) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) chain = chain.run_cmd( f"pip install --no-cache-dir --find-links {_WHEELHOUSE_PATH} " f"{requirements}", diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index 6872934..abf6f40 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -147,7 +147,7 @@ from ..files import files_step from ..redact import redact from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in -from ..resolve_conda import conda_lock_protected_pins, is_conda_lock +from ..resolve_conda import conda_lock_pip_requirements, is_conda_lock from ..spec import GPU_SIZE_CLASSES, BuildSecret, Environment, command_names_secret from .managed import ManagedBuilder @@ -214,14 +214,16 @@ def _scrubbed(text: str, values: dict[str, str]) -> str: def _install_packages(image: Any, lock_text: str) -> Any: """The package layer for this lock: a conda source (E3-02) installs the - `@EXPLICIT` lock with Modal's own `micromamba_install` and layers the - protected pip pins the resolver forced (from the lock's own - `# datalayer-protected:` header); a pip source runs `uv pip sync`.""" + `@EXPLICIT` lock with Modal's own `micromamba_install` — which brings + micromamba itself, so no bootstrap is needed here — and layers the pip + layer the solve resolved (the user's pip requirements and the protected + pins over them, from the lock's own `# datalayer-pip:` header); a pip + source runs `uv pip sync`.""" if is_conda_lock(lock_text): image = image.micromamba_install(spec_file=_LOCK_PATH) - pins = conda_lock_protected_pins(lock_text) - if pins: - image = image.pip_install(*pins, find_links=WHEELHOUSE_IMAGE_PATH) + pip_requirements = conda_lock_pip_requirements(lock_text) + if pip_requirements: + image = image.pip_install(*pip_requirements, find_links=WHEELHOUSE_IMAGE_PATH) return image return image.run_commands( f'pip install --no-cache-dir "uv=={_UV_VERSION}"', diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 9e408a1..5a9fed6 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -49,7 +49,7 @@ from datetime import datetime, timezone from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Protocol +from typing import TYPE_CHECKING, Any, Callable, Protocol from .bases import APPROVED_BASES, ApprovedBase, channel_snapshot, resolve_base from .build_secrets import resolve_build_secret @@ -76,6 +76,9 @@ parse_requirements_txt, ) +if TYPE_CHECKING: + from .resolve_conda import CondaResolveRunner + __all__ = [ "APT_PIN_PREFIX", "APT_SNAPSHOT_PREFIX", @@ -1158,6 +1161,7 @@ def resolve_environment( credential: Any = None, log: Callable[[str], None] | None = None, runner: ResolveRunner | None = None, + conda_runner: CondaResolveRunner | None = None, bases: dict[str, ApprovedBase] = APPROVED_BASES, resolved_at: datetime | None = None, uv: str | None = None, @@ -1185,6 +1189,12 @@ def resolve_environment( Where the solve's output goes, line by line: the build's log. runner Where the solve runs. D-9's BuildKit solve by default. + conda_runner + Where a conda ``dependencyFile``'s own ``micromamba`` solve runs + (E3-02): the conda seam's runner, injected by tests and by ``plane + local`` the same way ``runner`` is for a pip source, and D-9's BuildKit + conda solve by default. A pip source ignores it, and a conda source + ignores ``runner``, since the two solves are different tools. bases The approved bases, injected by tests and by a plane whose channel is published somewhere else. @@ -1263,6 +1273,7 @@ def resolve_environment( resolved_bases=resolved_bases, credential=credential, log=say, + runner=conda_runner, resolved_at=resolved_at, ) if dependency_file.source_format == "pyproject": diff --git a/code_sandboxes/environments/resolve_conda.py b/code_sandboxes/environments/resolve_conda.py index 362de55..8a7fc94 100644 --- a/code_sandboxes/environments/resolve_conda.py +++ b/code_sandboxes/environments/resolve_conda.py @@ -64,7 +64,6 @@ EnvironmentsError, ) from .resolve import ( - PROTECTED_PIN_PREFIX, WHEELHOUSE_IMAGE_PATH, WHEELHOUSE_PATH, MergedRequirements, @@ -81,7 +80,7 @@ "CondaResolveRunner", "MicromambaResolveRunner", "conda_lock_document", - "conda_lock_protected_pins", + "conda_lock_pip_requirements", "explicit_lock_packages", "is_conda_lock", "merge_conda_pip", @@ -98,12 +97,56 @@ #: ``linux/amd64``, so the solve is for ``linux-64`` in conda's own naming. CONDA_PLATFORM = "linux-64" -#: How the protected pip pins are recorded in the lock's header, the same -#: prefix :func:`code_sandboxes.environments.resolve.lock_document` uses, so a -#: reader of either lock finds Datalayer's pins the same way. +#: The marker the ``@EXPLICIT`` conda lock body opens with, and the key the +#: ``environment.yml`` names its pip layer under. _EXPLICIT_MARKER = "@EXPLICIT" _PIP_SECTION_KEY = "pip" +#: How the whole pip layer is recorded in the lock's header — the user's pip +#: requirements *and* the protected pins Datalayer forces over them, pinned to +#: the versions the solve resolved — one ``# datalayer-pip: `` line each, +#: above the ``@EXPLICIT`` body. A builder installs the conda layer from the +#: body and then this pip layer, so the whole of what a version resolved to is +#: in the one document and nothing resolved in the solve is lost from the build. +CONDA_PIP_PREFIX = "# datalayer-pip: " + +#: The ``micromamba`` the conda solve and every conda build use, pinned so the +#: tool that resolves is the tool that installs (E3-02): the approved base bakes +#: uv, the wheelhouse and the doctor, but not micromamba, so it is brought in +#: here rather than assumed. A BuildKit build copies the binary from this image; +#: a builder driving an SDK installs the same pinned release. +MICROMAMBA_VERSION = "2.0.5" +MICROMAMBA_IMAGE = f"mambaorg/micromamba:{MICROMAMBA_VERSION}" +MICROMAMBA_BINARY = "/usr/local/bin/micromamba" + +#: A channel URL that carries a credential in its userinfo — the same shape +#: :func:`code_sandboxes.environments.spec._index_findings` refuses in an index +#: URL, so a token is caught the same way whichever field names it. +_URL_CREDENTIALS = re.compile(r"^[a-z][a-z0-9+.-]*://[^/@\s]+:[^/@\s]*@", re.IGNORECASE) + + +def micromamba_bootstrap_dockerfile_line() -> str: + """The Dockerfile line that brings the pinned micromamba into a build. + + ``COPY --from`` the pinned micromamba image, so the binary is present and + reproducible without a network fetch inside the build itself. Used by the + resolver's own solve image and by the Datalayer (BuildKit) builder. + """ + return f"COPY --from={MICROMAMBA_IMAGE} /bin/micromamba {MICROMAMBA_BINARY}" + + +def micromamba_bootstrap_command() -> str: + """The shell command that installs the pinned micromamba into a build. + + For a builder that drives an SDK (E2B, Daytona) rather than emitting a + Dockerfile: the same pinned release ``COPY --from`` brings, fetched into + ``/usr/local/bin`` so a later ``micromamba install`` finds it on the PATH. + """ + return ( + f"curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/{MICROMAMBA_VERSION} " + "| tar -xj -C /usr/local/bin --strip-components=1 bin/micromamba" + ) + def _utcnow() -> datetime: return datetime.now(timezone.utc) @@ -238,6 +281,17 @@ def _channels(document: Mapping[str, Any]) -> tuple[str, ...]: ) text = channel.strip() if text: + if _URL_CREDENTIALS.match(text): + # The same refusal `spec.packages.python.indexes` gives a + # credential-bearing index URL: a token in the channel is + # copied into the solve context and can reappear in the + # explicit lock, so it belongs in the build's secrets, not here. + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry carries a credential in its URL; reference the " + "credential in `buildSecrets`", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) channels.append(text) return tuple(channels) @@ -316,9 +370,50 @@ class CondaResolveRequest: @dataclass class CondaResolveOutcome: - """A conda solve's answer: the explicit lock, verbatim from ``micromamba``.""" + """A conda solve's answer: the explicit lock, and the pip layer it resolved. + + ``lock_text`` is the ``@EXPLICIT`` conda lock, verbatim from ``micromamba``. + ``pip_lock`` is the pip layer the same solve installed — the user's pip + requirements and the protected pins forced over them — pinned to the + versions it resolved, read back from ``micromamba env export`` so the + artifact carries the whole of what the solve produced, not the conda layer + alone (E3-02). A runner that cannot read the prefix back leaves it empty, + and :func:`conda_lock_document` falls back to the merged requirements. + """ lock_text: str + pip_lock: tuple[str, ...] = () + + +def pip_requirements_from_env_yaml(text: str) -> tuple[str, ...]: + """The pip layer a ``micromamba env export`` names, pinned, in order. + + A conda ``env export`` (the YAML form, not ``--explicit``) lists the pip + packages it installed under a single ``{"pip": [...]}`` entry of its + ``dependencies``, each ``name==version`` — cleanly separated from the conda + packages, which are their own strings. This reads that section back, so the + solve's resolved pip versions become the lock's pip layer. A malformed or + pip-less export is an empty layer, never a raised error: the explicit lock + is what a solve is judged by, and its own marker is checked elsewhere. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError: + return () + if not isinstance(document, Mapping): + return () + dependencies = document.get("dependencies") + if not isinstance(dependencies, Sequence) or isinstance(dependencies, (str, bytes)): + return () + requirements: list[str] = [] + for entry in dependencies: + if isinstance(entry, Mapping) and _PIP_SECTION_KEY in entry: + for requirement in entry[_PIP_SECTION_KEY] or []: + if isinstance(requirement, str) and requirement.strip(): + requirements.append(requirement.strip()) + return tuple(requirements) class CondaResolveRunner(Protocol): @@ -452,18 +547,51 @@ def solve( for line in (created.stderr or "").splitlines(): say(line) raise parse_conda_failure(created.stderr or created.stdout or "") - export = subprocess.run( # noqa: S603 - the argv is built here + export = self._export( [self._micromamba, "env", "export", "--explicit", "--prefix", str(prefix)], + say, + ) + # The pip layer the same solve installed, pinned, read back from the + # YAML export's own `pip:` section (E3-02): the explicit export above + # carries conda packages alone, so without this the user's resolved + # pip requirements would be absent from the artifact. + pip_export = self._export( + [self._micromamba, "env", "export", "--prefix", str(prefix)], + say, + ) + return CondaResolveOutcome( + lock_text=export.stdout, + pip_lock=pip_requirements_from_env_yaml(pip_export.stdout), + ) + + def _export( + self, argv: list[str], say: Callable[[str], None] + ) -> subprocess.CompletedProcess[str]: + """One ``micromamba env export``, its timeout handled the same as the solve. + + The ``create`` above and both exports share the one refusal so a timeout + anywhere becomes ``DL_ENV_PROVIDER_ERROR`` rather than a raw + :class:`subprocess.TimeoutExpired` a caller cannot classify or retry. + """ + try: + result = subprocess.run( # noqa: S603 - the argv is built here + argv, capture_output=True, text=True, timeout=self._timeout, check=False, ) - if export.returncode != 0: - for line in (export.stderr or "").splitlines(): - say(line) - raise parse_conda_failure(export.stderr or export.stdout or "") - return CondaResolveOutcome(lock_text=export.stdout) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if result.returncode != 0: + for line in (result.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(result.stderr or result.stdout or "") + return result class BuildkitCondaResolveRunner: @@ -510,21 +638,28 @@ def dockerfile(self, request: CondaResolveRequest) -> str: into the context, never interpolated into a shell command — and the wheelhouse is brought along for the one protected pin no index has (E1-04), reached through ``PIP_FIND_LINKS`` the same way the local - runner reaches it. + runner reaches it. ``micromamba`` is copied in from its pinned image + first, because the approved base bakes uv and the wheelhouse but not it. + The explicit lock and the pip layer are both exported, so the artifact + carries the whole of what the solve resolved, not the conda layer alone. """ + micromamba = shlex.quote(MICROMAMBA_BINARY) return "\n".join( [ f"FROM {request.base_reference} AS solve", "USER root", "WORKDIR /solve", + micromamba_bootstrap_dockerfile_line(), "COPY environment.yml ./environment.yml", "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), "RUN --mount=type=cache,target=/opt/conda/pkgs " - "micromamba create --yes --prefix /solve/prefix " + f"{micromamba} create --yes --prefix /solve/prefix " f"--platform {shlex.quote(request.platform)} --file environment.yml", - "RUN micromamba env export --explicit --prefix /solve/prefix > /solve/lock.txt", + f"RUN {micromamba} env export --explicit --prefix /solve/prefix > /solve/lock.txt", + f"RUN {micromamba} env export --prefix /solve/prefix > /solve/pip-env.yml", "FROM scratch", "COPY --from=solve /solve/lock.txt /lock.txt", + "COPY --from=solve /solve/pip-env.yml /pip-env.yml", ] ) + "\n" @@ -584,7 +719,13 @@ def solve( if finished.returncode != 0: raise parse_conda_failure(finished.stderr or finished.stdout or "") lock = (out / "lock.txt").read_text(encoding="utf-8") - return CondaResolveOutcome(lock_text=lock) + pip_env = out / "pip-env.yml" + pip_lock = ( + pip_requirements_from_env_yaml(pip_env.read_text(encoding="utf-8")) + if pip_env.exists() + else () + ) + return CondaResolveOutcome(lock_text=lock, pip_lock=pip_lock) def _environment(self, request: CondaResolveRequest) -> dict[str, str] | None: auth = dict(request.registry_auth or {}) @@ -631,23 +772,25 @@ def is_conda_lock(lock_text: str | None) -> bool: return any(line.strip() == _EXPLICIT_MARKER for line in lock_text.splitlines()) -def conda_lock_protected_pins(lock_text: str) -> list[str]: - """The pip requirements a conda lock's header records as Datalayer's pins. +def conda_lock_pip_requirements(lock_text: str) -> list[str]: + """The whole pip layer a conda lock's header records, in order. - :func:`conda_lock_document` writes the protected pip pins as - ``# datalayer-protected: `` lines above the ``@EXPLICIT`` body. A - builder installs the conda layer from the body and then this pip layer, so - the kernel stack (E1-04) is present the same way it is for every source. + :func:`conda_lock_document` writes the pip layer the solve resolved as + ``# datalayer-pip: `` lines above the ``@EXPLICIT`` body: the user's + own pip requirements and the protected pins Datalayer forced over them, + pinned to the versions the solve produced. A builder installs the conda + layer from the body and then this pip layer, so the whole of what the + version resolved to is built and nothing the solve installed is lost (E3-02). """ - prefix = PROTECTED_PIN_PREFIX.strip() - pins: list[str] = [] + prefix = CONDA_PIP_PREFIX.strip() + requirements: list[str] = [] for raw in lock_text.splitlines(): line = raw.strip() if line.startswith(prefix): requirement = line[len(prefix) :].strip() if requirement: - pins.append(requirement) - return pins + requirements.append(requirement) + return requirements def conda_lock_document( @@ -661,11 +804,15 @@ def conda_lock_document( ) -> dict[str, Any]: """The stored conda lock: its text, its digest, and what a reader needs. - The protected pins are written as comments above the explicit lock, the - same ``# datalayer-protected:`` lines the pip lock carries, so the one - document says the whole of what a build installs — the conda packages by - URL and hash, and the pip pins Datalayer forced over the pip layer — while - staying a file ``micromamba create --file`` reads unchanged. + The pip layer the solve resolved is written as comments above the explicit + lock — the ``# datalayer-pip:`` lines a builder installs after the conda + packages — so the one document says the whole of what a build installs: the + conda packages by URL and hash, and the user's pip requirements with the + protected pins Datalayer forced over them, pinned to the versions the solve + produced. It stays a file ``micromamba create --file`` reads unchanged. + The pip layer is the solve's own (``outcome.pip_lock``) when the runner + could read the prefix back, and the merged requirements otherwise, so it is + always complete rather than the protected pins alone. """ when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() header = [ @@ -675,8 +822,9 @@ def conda_lock_document( f"# platform: {platform}", f"# base: {base_reference}", ] - for constraint in merged.constraints: - header.append(f"{PROTECTED_PIN_PREFIX}{constraint}") + pip_layer = list(outcome.pip_lock) or list(merged.requirements) + for requirement in pip_layer: + header.append(f"{CONDA_PIP_PREFIX}{requirement}") body = outcome.lock_text.lstrip("\n") if _EXPLICIT_MARKER not in {line.strip() for line in body.splitlines()}: raise EnvironmentsError( diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index f8a804c..4bec78b 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -194,6 +194,47 @@ def index_is_public(url: str) -> bool: return _package_index_host(url) in PUBLIC_PACKAGE_INDEX_HOSTS +#: The conda channels D-12 counts as public: a published conda version +#: (E3-02's ``dependencyFile``) may resolve only from these, since a private +#: channel is reached with a token no public reader holds — the same boundary +#: :data:`PUBLIC_PACKAGE_INDEX_HOSTS` draws for pip indexes. The bare names +#: anaconda.org serves openly, and the hosts a channel URL may name; any other +#: name or host is private and blocks publication. +PUBLIC_CONDA_CHANNELS = frozenset( + { + "conda-forge", + "bioconda", + "defaults", + "nodefaults", + "main", + "r", + "anaconda", + "pkgs/main", + "pkgs/r", + "msys2", + } +) +PUBLIC_CONDA_CHANNEL_HOSTS = frozenset( + {"conda.anaconda.org", "repo.anaconda.com", "anaconda.org"} +) + + +def channel_is_public(channel: str) -> bool: + """Whether a conda channel is one D-12 lets a published version resolve from. + + A channel is a URL, whose host must be a public conda host, or a bare name, + which is public only when it is one of the well-known open channels — an + unlisted name (say a private org's) is treated as private, since a bare name + on anaconda.org may still need a token the public does not have. + """ + text = channel.strip() + if not text: + return True + if "://" in text: + return _package_index_host(text) in PUBLIC_CONDA_CHANNEL_HOSTS + return text.lower() in PUBLIC_CONDA_CHANNELS + + class PythonPackages(_Model): manager: Literal["uv", "pip", "conda"] = "uv" dependencies: list[str] = Field(default_factory=list) @@ -907,9 +948,46 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: PUBLICATION_BLOCKED, ) ) + private_channels = [ + channel for channel in _conda_channels(environment) if not channel_is_public(channel) + ] + if private_channels: + findings.append( + SpecFinding( + "spec.build.dependencyFile.content.channels", + "a version that resolves from a private conda channel " + f"({', '.join(private_channels)}) can never be published to the public " + "Library (D-12); publish only from public channels " + f"({', '.join(sorted(PUBLIC_CONDA_CHANNELS))})", + PUBLICATION_BLOCKED, + ) + ) return findings +def _conda_channels(environment: Environment) -> tuple[str, ...]: + """The channels a conda ``dependencyFile`` names, or none for any other source. + + A conda ``environment.yml``'s ``channels`` are package inputs the same as a + pip source's indexes, so publication weighs them the same (D-12). A file + that will not parse has no channels to weigh here — validation refuses it + before it is ever published — so a parse failure is an empty tuple, not a + raise. + """ + build = environment.spec.build + dependency_file = build.dependency_file + if build.source != "dependencyFile" or dependency_file is None: + return () + if dependency_file.source_format != "conda": + return () + from .resolve_conda import parse_conda_environment + + try: + return parse_conda_environment(dependency_file.content).channels + except EnvironmentsError: + return () + + def assert_publishable(environment: Environment) -> None: """Raise ``DL_ENV_PUBLICATION_BLOCKED`` unless this version may be published (D-12). diff --git a/docs/docs/environments/specification.mdx b/docs/docs/environments/specification.mdx index bd048b4..c28ea8a 100644 --- a/docs/docs/environments/specification.mdx +++ b/docs/docs/environments/specification.mdx @@ -27,7 +27,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `base.ref` | — | An approved base: `datalayer/python-cpu` or `datalayer/python-cuda`. | | `base.channel` | — | The release channel, resolved to a digest per variant when the version is resolved. | | `platform.architecture` | `linux/amd64` | Only `linux/amd64`. | -| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. `conda` is not buildable yet. | +| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. A `conda` **`packages`** list is not built yet — bring a conda `environment.yml` as a `dependencyFile` instead (see `build.source`). | | `packages.python.dependencies` | none | PEP 508 requirements. | | `packages.python.constraints` | none | PEP 508 requirements, applied under Datalayer's protected constraints, which win. | | `packages.python.indexes` | `https://pypi.org/simple` | `https` URLs, with no credential in them. | @@ -42,7 +42,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. | | `compatibility.variants.optional` | none | The same variants, none of them also required. | | `compatibility.regions` | none | Region names. | -| `build.source` | `packages` | `packages`, `dependencyFile` (a `requirements.txt`, or a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved) and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base) all resolve on the Datalayer variant; `image` from a private registry, and `dockerfile`, are refused until they do. | +| `build.source` | `packages` | `packages`, `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer) and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base) all resolve on the Datalayer variant; `image` from a private registry, and `dockerfile`, are refused until they do. | ## The spec digest diff --git a/pyproject.toml b/pyproject.toml index b005720..f571e28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "jupyter-server", "jupyter-server-client", "pydantic>=2.0", + "pyyaml", "rich", "typer>=0.12.0", ] @@ -75,7 +76,7 @@ test = [ "pytest-cov>=4.0", ] lint = ["mdformat>0.7", "mdformat-gfm>=0.3.5", "ruff"] -typing = ["mypy>=0.990"] +typing = ["mypy>=0.990", "types-PyYAML"] [project.license] file = "LICENSE" diff --git a/tests/test_environment_datalayer_builder.py b/tests/test_environment_datalayer_builder.py index 70b8c7b..c68e557 100644 --- a/tests/test_environment_datalayer_builder.py +++ b/tests/test_environment_datalayer_builder.py @@ -44,12 +44,13 @@ LOCK_DIGEST = "sha256:" + "dd" * 32 #: A conda explicit lock (E3-02): the `@EXPLICIT` marker, one conda package -#: URL, and the protected pip pins the resolver forced over the pip layer. +#: URL, and the pip layer the solve resolved (the user's pip requirements and +#: the protected pins over them). CONDA_LOCK = ( "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" "# python: 3.13\n" "# platform: linux-64\n" - "# datalayer-protected: ipykernel==7.3.0\n" + "# datalayer-pip: ipykernel==7.3.0\n" "@EXPLICIT\n" "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" ) @@ -271,17 +272,20 @@ def test_it_installs_from_the_lock_with_hashes(self) -> None: assert "geopandas==1.1.1" not in dockerfile def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: - """A conda source (E3-02): the `@EXPLICIT` lock installs with - `micromamba`, and the protected pip pins the resolver forced over the - pip layer follow, so the kernel stack (E1-04) is present the same.""" + """A conda source (E3-02): the `@EXPLICIT` lock installs with a pinned + `micromamba` copied in first, and the pip layer the solve resolved + follows, so the kernel stack (E1-04) is present the same.""" request = a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC) dockerfile = a_builder().dockerfile(request) assert ( "micromamba install --yes --name base --file /opt/datalayer/lock.txt" in dockerfile ) - # The header's own protected pin, installed with pip after the conda layer. - pip = dockerfile.index("uv pip install --system") + # micromamba is copied in from its pinned image before it is invoked. + bootstrap = dockerfile.index("COPY --from=mambaorg/micromamba") micromamba = dockerfile.index("micromamba install") + assert bootstrap < micromamba + # The header's own pip layer, installed with pip after the conda layer. + pip = dockerfile.index("uv pip install --system") assert micromamba < pip assert "ipykernel==7.3.0" in dockerfile # A conda source never runs the pip-lock `uv pip sync`. diff --git a/tests/test_environment_daytona_builder.py b/tests/test_environment_daytona_builder.py index b468296..5ba666c 100644 --- a/tests/test_environment_daytona_builder.py +++ b/tests/test_environment_daytona_builder.py @@ -50,7 +50,7 @@ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" "# python: 3.13\n" "# platform: linux-64\n" - "# datalayer-protected: ipykernel==7.3.0\n" + "# datalayer-pip: ipykernel==7.3.0\n" "@EXPLICIT\n" "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" ) @@ -479,16 +479,17 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--find-links /opt/datalayer/wheelhouse" in sync.args[0] def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: - """A conda source (E3-02): `micromamba install --file` reads the - `@EXPLICIT` lock, and the protected pip pins the resolver forced over - the pip layer follow — never the pip-lock `uv pip sync`.""" + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" daytona = FakeDaytonaModule() a_builder(daytona=daytona).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) image = daytona.client.snapshot.create_calls[0].args[0].image runs = calls_named(image, "run_commands") + bootstrap = next(i for i, call in enumerate(runs) if "micro.mamba.pm" in call.args[0]) micromamba = next(i for i, call in enumerate(runs) if "micromamba install" in call.args[0]) pip = next(i for i, call in enumerate(runs) if "ipykernel==7.3.0" in call.args[0]) - assert micromamba < pip + assert bootstrap < micromamba < pip assert not any("uv pip sync" in call.args[0] for call in runs) def test_user_root_brackets_the_install_steps(self) -> None: diff --git a/tests/test_environment_e2b_builder.py b/tests/test_environment_e2b_builder.py index 9179fc8..c043b4e 100644 --- a/tests/test_environment_e2b_builder.py +++ b/tests/test_environment_e2b_builder.py @@ -40,7 +40,7 @@ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" "# python: 3.13\n" "# platform: linux-64\n" - "# datalayer-protected: ipykernel==7.3.0\n" + "# datalayer-pip: ipykernel==7.3.0\n" "@EXPLICIT\n" "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" ) @@ -410,11 +410,16 @@ def test_no_apt_step_when_the_lock_pins_none(self) -> None: assert not any(call.name == "run_cmd" and "apt-get" in call.args[0] for call in fake.calls) def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: - """A conda source (E3-02): `micromamba install --file` reads the - `@EXPLICIT` lock, and the protected pip pins the resolver forced over - the pip layer follow — never the pip-lock `uv pip sync`.""" + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" fake = FakeTemplate() a_builder(fake).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + bootstrap = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micro.mamba.pm" in call.args[0] + ) micromamba = next( i for i, call in enumerate(fake.calls) @@ -425,7 +430,7 @@ def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> No for i, call in enumerate(fake.calls) if call.name == "run_cmd" and "ipykernel==7.3.0" in call.args[0] ) - assert micromamba < pip + assert bootstrap < micromamba < pip assert not any( call.name == "run_cmd" and "uv pip sync" in call.args[0] for call in fake.calls ) diff --git a/tests/test_environment_modal_builder.py b/tests/test_environment_modal_builder.py index fcf0001..0adc96f 100644 --- a/tests/test_environment_modal_builder.py +++ b/tests/test_environment_modal_builder.py @@ -51,7 +51,7 @@ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" "# python: 3.13\n" "# platform: linux-64\n" - "# datalayer-protected: ipykernel==7.3.0\n" + "# datalayer-pip: ipykernel==7.3.0\n" "@EXPLICIT\n" "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" ) @@ -517,8 +517,8 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: """A conda source (E3-02): Modal's own `micromamba_install` reads the - `@EXPLICIT` lock, and `pip_install` layers the protected pip pins the - resolver forced — never the pip-lock `uv pip sync`.""" + `@EXPLICIT` lock, and `pip_install` layers the pip layer the solve + resolved — never the pip-lock `uv pip sync`.""" modal = FakeModalModule() a_builder(modal=modal).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) [image] = modal.Image.created diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 43af27e..79d3253 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -454,6 +454,54 @@ def test_conda_waits_for_its_own_solver(self) -> None: assert raised.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" assert raised.value.detail["manager"] == "conda" + def test_a_conda_dependency_file_uses_the_conda_runner_it_is_given(self) -> None: + """The main API forwards its `conda_runner` to the conda solve, so a + local or test solver reaches it the same way `runner` reaches pip.""" + from code_sandboxes.environments.resolve_conda import ( + CondaResolveOutcome, + CondaResolveRequest, + ) + + class RecordedConda: + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request, log=None): # type: ignore[no-untyped-def] + self.request = request + return self._outcome + + conda_runner = RecordedConda( + CondaResolveOutcome( + lock_text=( + "# platform: linux-64\n@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + + "bb" * 32 + + "\n" + ) + ) + ) + spec = a_spec( + packages={}, + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "channels:\n - conda-forge\ndependencies:\n - gdal=3.9\n", + }, + }, + ) + resolve_environment( + spec=spec, + variants=["datalayer"], + conda_runner=conda_runner, + bases=BASES, + ) + assert conda_runner.request is not None + assert "gdal=3.9" in conda_runner.request.environment_yml + def test_the_credentials_registry_auth_reaches_the_runner(self) -> None: class Credential: def registry_auth(self) -> dict[str, str]: diff --git a/tests/test_environment_resolve_conda.py b/tests/test_environment_resolve_conda.py index 1339f22..fa4d414 100644 --- a/tests/test_environment_resolve_conda.py +++ b/tests/test_environment_resolve_conda.py @@ -11,6 +11,7 @@ from __future__ import annotations +import subprocess from datetime import datetime, timezone import pytest @@ -29,6 +30,7 @@ merge_conda_pip, parse_conda_environment, parse_conda_failure, + pip_requirements_from_env_yaml, rendered_environment, resolve_conda_environment, ) @@ -174,6 +176,53 @@ def test_channels_that_are_not_a_list_are_refused(self) -> None: with pytest.raises(EnvironmentsError): parse_conda_environment("channels: conda-forge\ndependencies:\n - gdal\n") + def test_a_channel_url_carrying_a_credential_is_refused(self) -> None: + # The same refusal a credential-bearing pip index gets: the token + # belongs in the build's secrets, never verbatim in the spec. + text = ( + "channels:\n" + " - https://user:tok@conda.example.com/private\n" + "dependencies:\n - gdal\n" + ) + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert "credential" in caught.value.message + assert caught.value.detail["field"].endswith("channels[0]") + + def test_a_plain_channel_url_without_a_credential_is_kept(self) -> None: + env = parse_conda_environment( + "channels:\n - https://conda.anaconda.org/conda-forge\n" + "dependencies:\n - gdal\n" + ) + assert env.channels == ("https://conda.anaconda.org/conda-forge",) + + +class TestReadingThePipExport: + def test_it_reads_the_pip_section_in_order(self) -> None: + export = ( + "name: solved\n" + "channels:\n - conda-forge\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal=3.9.2\n" + " - pip:\n" + " - shapely==2.0.6\n" + " - ipykernel==7.3.0\n" + ) + assert pip_requirements_from_env_yaml(export) == ( + "shapely==2.0.6", + "ipykernel==7.3.0", + ) + + def test_an_export_without_a_pip_section_is_an_empty_layer(self) -> None: + export = "dependencies:\n - python=3.13\n - gdal=3.9.2\n" + assert pip_requirements_from_env_yaml(export) == () + + def test_a_malformed_export_is_an_empty_layer_never_a_raise(self) -> None: + assert pip_requirements_from_env_yaml("dependencies: [\n") == () + assert pip_requirements_from_env_yaml("- just\n- a\n- list\n") == () + # -- Datalayer's pins over the pip layer ------------------------------------- @@ -252,7 +301,10 @@ def test_the_document_records_the_pins_and_is_deterministic(self) -> None: ) assert document["format"] == CONDA_LOCK_FORMAT assert document["package_count"] == 2 - assert "# datalayer-protected: ipykernel==7.3.0" in document["content"] + # The header carries the complete pip layer: the user's own pip + # requirement and the protected pin the resolver forced over it. + assert "# datalayer-pip: shapely==2.0.6" in document["content"] + assert "# datalayer-pip: ipykernel==7.3.0" in document["content"] assert "@EXPLICIT" in document["content"] again = conda_lock_document( CondaResolveOutcome(lock_text=EXPLICIT_LOCK), @@ -286,6 +338,28 @@ def test_the_local_runner_refuses_without_micromamba(self) -> None: runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + def test_an_export_that_times_out_is_a_provider_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The solve finishes, but the `env export` that reads the lock back + # exceeds the deadline: its timeout is classified the same as the + # solve's, never left as a raw subprocess.TimeoutExpired. + runner = MicromambaResolveRunner(micromamba="/usr/local/bin/micromamba", timeout=5.0) + calls = {"n": 0} + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls["n"] += 1 + if calls["n"] == 1: + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + raise subprocess.TimeoutExpired(argv, 5.0) + + monkeypatch.setattr( + "code_sandboxes.environments.resolve_conda.subprocess.run", fake_run + ) + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + def test_the_buildkit_runner_refuses_without_buildctl(self) -> None: runner = BuildkitCondaResolveRunner(buildctl="") with pytest.raises(EnvironmentsError) as caught: @@ -320,6 +394,11 @@ def test_the_buildkit_dockerfile_brings_the_wheelhouse_and_solves(self) -> None: assert "micromamba create" in dockerfile assert "PIP_FIND_LINKS" in dockerfile assert "micromamba env export --explicit" in dockerfile + # The pinned micromamba is copied in (the base bakes uv, not it), and + # the pip layer is exported alongside the explicit lock. + assert "COPY --from=mambaorg/micromamba" in dockerfile + assert "env export --prefix /solve/prefix > /solve/pip-env.yml" in dockerfile + assert "COPY --from=solve /solve/pip-env.yml /pip-env.yml" in dockerfile # -- The whole resolve, through the recorded runner -------------------------- diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index 3cb8bc3..af3a44a 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -613,3 +613,32 @@ def test_only_public_indexes_are_publishable(self) -> None: ] environment = parse_environment(data) assert publication_findings(environment) == [] + + def test_a_private_conda_channel_blocks_publication(self) -> None: + """D-12 the same for a conda source: a channel reached with a + credential the public does not hold can never be published.""" + data = a_dependency_file_document( + sourceFormat="conda", + content=( + "channels:\n" + " - conda-forge\n" + " - https://conda.mycorp.internal/private\n" + "dependencies:\n - gdal\n" + ), + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.build.dependencyFile.content.channels" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "conda.mycorp.internal" in findings[0].message + + def test_public_conda_channels_are_publishable(self) -> None: + data = a_dependency_file_document( + sourceFormat="conda", + content="channels:\n - conda-forge\n - bioconda\ndependencies:\n - gdal\n", + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + assert publication_findings(environment) == [] From 1704b36a723da15d125ee20077e97e17b79b661b Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Tue, 15 Sep 2026 10:24:02 +0200 Subject: [PATCH 03/22] envs --- code_sandboxes/environments/__init__.py | 8 ++ code_sandboxes/environments/contract.py | 98 ++++++++++++++++++++++++- code_sandboxes/environments/spec.py | 12 +-- tests/test_environment_contract.py | 51 +++++++++++++ tests/test_environment_spec.py | 15 +++- 5 files changed, 174 insertions(+), 10 deletions(-) diff --git a/code_sandboxes/environments/__init__.py b/code_sandboxes/environments/__init__.py index fa98607..f3cb16e 100644 --- a/code_sandboxes/environments/__init__.py +++ b/code_sandboxes/environments/__init__.py @@ -49,8 +49,12 @@ CONTRACT_V1, SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + BuildContextFinding, SandboxContract, + check_build_context, check_dockerfile, + validate_build_context, validate_dockerfile, ) from .errors import ERROR_CODES, EnvironmentsError, ErrorCode, map_provider_error @@ -91,6 +95,8 @@ "ApprovedBase", "ArtifactReference", "Attestor", + "BuildContextEntry", + "BuildContextFinding", "BuildRequest", "BuildkitResolveRunner", "CapabilityReport", @@ -114,6 +120,7 @@ "can_transition", "canonical_digest", "canonical_json", + "check_build_context", "check_dockerfile", "decide", "fingerprint_matches", @@ -131,6 +138,7 @@ "spec_digest", "spec_findings", "transition", + "validate_build_context", "validate_dockerfile", "validate_environment", ] diff --git a/code_sandboxes/environments/contract.py b/code_sandboxes/environments/contract.py index 592c3c7..8df1fe8 100644 --- a/code_sandboxes/environments/contract.py +++ b/code_sandboxes/environments/contract.py @@ -24,26 +24,34 @@ import re import shlex import sys +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from pydantic import BaseModel, ConfigDict from .bases import is_approved_repository -from .errors import CAPABILITY_UNSUPPORTED, EnvironmentsError +from .errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError __all__ = [ "CONTRACT_V1", + "MAX_CONTEXT_FILES", + "MAX_CONTEXT_FILE_BYTES", + "MAX_CONTEXT_TOTAL_BYTES", "SANDBOX_CONTRACT_V1", "SUPPORTED_CONTRACTS", + "BuildContextEntry", + "BuildContextFinding", "ContractRow", "DockerfileFinding", "DockerfileInstruction", "SandboxContract", + "check_build_context", "check_dockerfile", "contract_markdown", "get_contract", "parse_dockerfile", + "validate_build_context", "validate_dockerfile", ] @@ -453,6 +461,94 @@ def check_dockerfile(text: str, *, contract: SandboxContract = SANDBOX_CONTRACT_ ) +# --- The build context (E3-03) ------------------------------------------------------- + +#: A `dockerfile` source uploads a build context to object storage. These bound +#: what Runtimes accepts before it issues a presigned URL, so a context cannot +#: be a way to smuggle a host file in (a symlink or `..`), or to fill a bucket. +MAX_CONTEXT_FILES = 2000 +MAX_CONTEXT_FILE_BYTES = 50 * 1024 * 1024 +MAX_CONTEXT_TOTAL_BYTES = 100 * 1024 * 1024 + +_CONTEXT_SEPARATOR = re.compile(r"[\\/]") + + +@dataclass(frozen=True) +class BuildContextEntry: + """One member of an uploaded build context: its path, size, and whether it is a symlink.""" + + path: str + size_bytes: int = 0 + is_symlink: bool = False + + +@dataclass(frozen=True) +class BuildContextFinding: + """Something in a build context that must not be uploaded.""" + + path: str + message: str + + def to_dict(self) -> dict[str, object]: + return {"path": self.path, "message": self.message} + + +def validate_build_context( + entries: Sequence[BuildContextEntry], +) -> list[BuildContextFinding]: + """Everything in a build context the upload refuses, in the order given. + + Refused: an absolute path, a `..` that would escape the context, a symlink + (which could point at a host file the build then reads), a file over the + per-file limit, and — once — a context with too many files or too many + bytes in all. + """ + findings: list[BuildContextFinding] = [] + total = 0 + for entry in entries: + path = entry.path + components = _CONTEXT_SEPARATOR.split(path) + if not path or all(part in ("", ".") for part in components): + findings.append(BuildContextFinding(path, "is not a path inside the context")) + elif path.startswith("/") or path.startswith("\\"): + findings.append(BuildContextFinding(path, "is an absolute path, not a context path")) + elif ".." in components: + findings.append(BuildContextFinding(path, "escapes the context with `..`")) + if entry.is_symlink: + findings.append( + BuildContextFinding(path, "is a symlink, which could read a host file") + ) + if entry.size_bytes > MAX_CONTEXT_FILE_BYTES: + findings.append( + BuildContextFinding( + path, f"is over the {MAX_CONTEXT_FILE_BYTES}-byte per-file limit" + ) + ) + total += entry.size_bytes + if len(entries) > MAX_CONTEXT_FILES: + findings.append( + BuildContextFinding("", f"has more than {MAX_CONTEXT_FILES} files") + ) + if total > MAX_CONTEXT_TOTAL_BYTES: + findings.append( + BuildContextFinding("", f"is over the {MAX_CONTEXT_TOTAL_BYTES}-byte total limit") + ) + return findings + + +def check_build_context(entries: Sequence[BuildContextEntry]) -> None: + """Refuse a build context the upload does not allow, naming the first fault.""" + findings = validate_build_context(entries) + if findings: + first = findings[0] + where = f"`{first.path}`: " if first.path else "" + raise EnvironmentsError( + SPEC_INVALID, + f"{where}{first.message}", + detail={"findings": [finding.to_dict() for finding in findings]}, + ) + + # --- The documentation page ---------------------------------------------------------- diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index 4bec78b..5d064b8 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -107,7 +107,7 @@ BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") #: What builds today; `dockerfile` is the one source still to come. -SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "image") +SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") SUPPORTED_PACKAGE_MANAGERS: tuple[str, ...] = ("uv", "pip") #: `requirements.txt` and `pyproject.toml`/`uv.lock` are archived on the #: version they resolved (E3-01); this bounds what a spec may carry inline, @@ -593,10 +593,12 @@ def spec_findings( ) ) - # An `image` source brings its own base (E3-04): `spec.base` names - # nothing Datalayer approved, so checking it against the table would - # refuse every import for the one reason imports exist to avoid. - if spec.build.source != "image": + # An `image` source brings its own base (E3-04), and a `dockerfile` + # source's base is the `FROM` its uploaded Dockerfile names (E3-03, + # validated against the approved bases by `check_dockerfile`, not here): + # `spec.base` names nothing Datalayer approved for either, so checking it + # against the table would refuse every one for the reason they exist. + if spec.build.source not in ("image", "dockerfile"): base = bases.get(spec.base.ref) if base is None: findings.append( diff --git a/tests/test_environment_contract.py b/tests/test_environment_contract.py index b669e0a..41ad1ef 100644 --- a/tests/test_environment_contract.py +++ b/tests/test_environment_contract.py @@ -16,11 +16,14 @@ from code_sandboxes.environments.contract import ( SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + check_build_context, check_dockerfile, contract_markdown, get_contract, main, parse_dockerfile, + validate_build_context, validate_dockerfile, ) from code_sandboxes.environments.doctor.datalayer_sandbox import ROW_IDS @@ -134,6 +137,54 @@ def test_the_parser_joins_continuations_and_skips_comments_inside_them() -> None assert instructions[1].arguments == "apt-get update && apt-get install -y gdal-bin" +# -- The build context (E3-03) ------------------------------------------------- + + +def test_a_plain_build_context_passes() -> None: + entries = [ + BuildContextEntry("Dockerfile", 200), + BuildContextEntry("src/app.py", 1024), + BuildContextEntry("data/model.bin", 5 * 1024 * 1024), + ] + assert validate_build_context(entries) == [] + check_build_context(entries) + + +@pytest.mark.parametrize( + ("entry", "message"), + [ + (BuildContextEntry("/etc/passwd", 10), "is an absolute path, not a context path"), + (BuildContextEntry("../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("a/../../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("link", is_symlink=True), "is a symlink, which could read a host file"), + ( + BuildContextEntry("big.bin", 50 * 1024 * 1024 + 1), + "is over the 52428800-byte per-file limit", + ), + ], +) +def test_a_forbidden_context_member_is_refused( + entry: BuildContextEntry, message: str +) -> None: + findings = validate_build_context([entry]) + assert any(finding.message == message for finding in findings), findings + with pytest.raises(EnvironmentsError) as refused: + check_build_context([entry]) + assert refused.value.code is errors.SPEC_INVALID + + +def test_too_many_files_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 1) for index in range(2001)] + findings = validate_build_context(entries) + assert any("more than 2000 files" in finding.message for finding in findings) + + +def test_too_many_bytes_in_all_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 40 * 1024 * 1024) for index in range(3)] + findings = validate_build_context(entries) + assert any("total limit" in finding.message for finding in findings) + + @pytest.mark.parametrize( ("image", "approved"), [ diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index af3a44a..88a8d9f 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -196,7 +196,6 @@ def _codes(data: dict[str, Any]) -> dict[str, str]: ("spec.contract", "sandbox-contract/v9", "spec.contract", UNSUPPORTED), ("spec.base.ref", "python", "spec.base.ref", INVALID), ("spec.language.version", "3.9", "spec.language.version", INVALID), - ("spec.build.source", "dockerfile", "spec.build.source", UNSUPPORTED), ("spec.packages.python.manager", "conda", "spec.packages.python.manager", UNSUPPORTED), # E3-05: a secret no postInstall command names would be mounted nowhere. ( @@ -307,8 +306,8 @@ def test_all_baked_files_together_are_capped() -> None: def test_an_invalid_field_outranks_something_unsupported() -> None: - data = mutated("spec.build.source", "dockerfile") - assert _codes(data) == {"spec.build.source": UNSUPPORTED} + data = mutated("spec.packages.python.manager", "conda") + assert _codes(data) == {"spec.packages.python.manager": UNSUPPORTED} with pytest.raises(EnvironmentsError) as unsupported: validate_environment(data) assert unsupported.value.code is errors.CAPABILITY_UNSUPPORTED @@ -319,10 +318,18 @@ def test_an_invalid_field_outranks_something_unsupported() -> None: assert invalid.value.code is errors.SPEC_INVALID assert {finding["field"] for finding in invalid.value.detail["findings"]} == { "metadata.name", - "spec.build.source", + "spec.packages.python.manager", } +def test_a_dockerfile_source_is_accepted_and_keeps_its_own_base() -> None: + # E3-03: the base is the `FROM` its uploaded Dockerfile names, so + # `spec.base` is not checked against the approved table (as for `image`). + data = mutated("spec.build.source", "dockerfile") + data["spec"]["base"]["ref"] = "python" + assert _codes(data) == {} + + # -- Dependency files (E3-01) -------------------------------------------------- From 916175925792ea48a1abe1777d578d212b805483 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 07:07:10 +0200 Subject: [PATCH 04/22] feat: swap the datalayer-kernels pin for jupyter-kernels from PyPI The pooled kernel manager moved to the public jupyter-kernels package (jupyter_kernels.pool.mapping.PooledMappingKernelManager, published as 1.2.23). PyPI serves it, so the protected pin resolves from the index and its wheel is dropped from the wheelhouse. Update the contract pin, the wheelhouse README, and the constraint/resolve tests. --- .../environments/constraints/sandbox-contract-v1.txt | 10 ++++++++++ .../environments/constraints/wheelhouse/README.md | 7 +++++++ tests/test_environment_constraints.py | 1 + tests/test_environment_resolve.py | 5 ++++- 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt index 3f27ac5..d5afa7c 100644 --- a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt +++ b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt @@ -20,8 +20,18 @@ # jupyterlab and jupyter-server-ydoc require jupyter-server>=2.19. The rebased fork # satisfies those ranges, so pip keeps it; its local label, which PyPI never serves, # tells it apart from the release. +# +# jupyter-kernels carries the pooled kernel manager the runtime's Jupyter config +# selects (kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager, +# plane/etc/dockerfiles/jupyter-python/etc/jupyter/config/jupyter_config.py). Unlike the +# deprecated private datalayer-kernels it replaced, PyPI serves it, so a resolve satisfies +# it from the index and the wheelhouse carries no wheel for it. The pin is still forced in: +# without it `uv pip sync --require-hashes` strips it from a user environment's image and the +# runtime's Jupyter server crashloops on a kernel_manager_class it can no longer import +# (found live on 2026-09-15, the first user-environment build to reach the smoke test). ipykernel==7.3.0 jupyter-client==8.9.1 jupyter-server==2.21.0+datalayer.1 jupyter-server-nbmodel==0.2.8 +jupyter-kernels==1.2.23 datalayer==1.7.4 diff --git a/code_sandboxes/environments/constraints/wheelhouse/README.md b/code_sandboxes/environments/constraints/wheelhouse/README.md index bbf2e79..2ed7334 100644 --- a/code_sandboxes/environments/constraints/wheelhouse/README.md +++ b/code_sandboxes/environments/constraints/wheelhouse/README.md @@ -38,3 +38,10 @@ Pure Python (`py3-none-any`), so one wheel serves every base's Python version this channel carries. Rebuild it here, under this same file name, whenever `services/kernels/Dockerfile`'s pinned commit changes — the constraints file's own version pin and this wheel move together. + +The pooled kernel manager pin, `jupyter-kernels`, needs no wheel here: PyPI +serves it, so a resolve satisfies it from the index like any other pin. It +was previously the private `datalayer-kernels`, which no index carried and +so was baked here as a wheel; the migration to the public +[`jupyter-kernels`](https://pypi.org/project/jupyter-kernels/) package +dropped that wheel. diff --git a/tests/test_environment_constraints.py b/tests/test_environment_constraints.py index 06dcef9..0aa1f0f 100644 --- a/tests/test_environment_constraints.py +++ b/tests/test_environment_constraints.py @@ -20,6 +20,7 @@ "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 79d3253..72e308d 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -141,10 +141,12 @@ def test_they_are_the_kernel_stack_at_one_version_each(self) -> None: "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } assert all(pin.version for pin in pins.values()) assert pins["jupyter-server"].version == "2.21.0+datalayer.1" + assert pins["jupyter-kernels"].version == "1.2.23" def test_a_requirement_that_agrees_with_a_pin_is_dropped_for_it(self) -> None: # The fork satisfies `>=2.19`, which is what jupyterlab asks for, so @@ -643,6 +645,7 @@ def a_pyproject_spec(**dependency_file: object) -> dict[str, object]: "jupyter-client==8.9.1 \\\n --hash=sha256:" + "cc" * 32 + "\n" "jupyter-server==2.21.0+datalayer.1 \\\n --hash=sha256:" + "dd" * 32 + "\n" "jupyter-server-nbmodel==0.2.8 \\\n --hash=sha256:" + "ee" * 32 + "\n" + "jupyter-kernels==1.2.23 \\\n --hash=sha256:" + "a7" * 32 + "\n" "datalayer==1.7.4 \\\n --hash=sha256:" + "ff" * 32 + "\n" ) @@ -658,7 +661,7 @@ def test_a_current_lock_is_exported_rather_than_resolved(self) -> None: pyproject_run=uv, ) assert answer["content"] == EXPORTED - assert answer["package_count"] == 6 + assert answer["package_count"] == 7 assert answer["python_version"] == "3.13" assert uv.calls[0][:2] == ["/usr/bin/uv", "lock"] assert uv.calls[1][:2] == ["/usr/bin/uv", "export"] From 2f1af3a30c9baccc43a337efc1caaba0f9032be0 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 07:39:20 +0200 Subject: [PATCH 05/22] release: 1.9.13 --- code_sandboxes/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 1e71576..014913a 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.12" +__version__ = "1.9.13" From c33ff7f489c8d31f61b4b718dad0856a440c452b Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 08:34:48 +0200 Subject: [PATCH 06/22] code-sandboxes 1.9.14: repin python-cpu 2026.09 to rebuilt base with jupyter-kernels --- CHANGELOG.md | 19 +++++++++++++++++++ code_sandboxes/__version__.py | 2 +- code_sandboxes/environments/bases.py | 9 +++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c79771c..de89de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,25 @@ ## Unreleased +## 1.9.14 + +- **`datalayer/python-cpu:2026.09` base channel repinned** to the rebuilt + `jupyter-python:0.2.2` (now carrying `jupyter-kernels==1.2.23`) plus the + contract layer, digest + `sha256:aa5413000bb5b6ecd0a0cf03959b107f0d572f65bf230c08bbdf9a4569775545`, + released 2026-09-16 to `environments/base/python-cpu`. Every variant pins the + same digest. + +## 1.9.13 + +- **`jupyter-kernels==1.2.23` forced into `sandbox-contract/v1`**: it carries + the pooled kernel manager the runtime's Jupyter config selects + (`kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager`), + replacing the deprecated private `datalayer-kernels`. PyPI serves it, so a + resolve satisfies it from the index and the wheelhouse carries no wheel for + it; the pin keeps `uv pip sync --require-hashes` from stripping it out of a + user environment's image. + ## 1.9.12 - **`owner_repository`, `owner_cache_repository` and `ECR_ENVIRONMENT_PREFIX` diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 014913a..193120b 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.13" +__version__ = "1.9.14" diff --git a/code_sandboxes/environments/bases.py b/code_sandboxes/environments/bases.py index 252ab1a..41f4669 100644 --- a/code_sandboxes/environments/bases.py +++ b/code_sandboxes/environments/bases.py @@ -109,8 +109,9 @@ def repository(self) -> str: base.ref: base for base in ( # E1-05: jupyter-python:0.2.2 (Ubuntu security packages and conda's own - # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped) plus the - # contract layer, released 2026-09-14 to environments/base/python-cpu. + # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped, jupyter-kernels + # installed) plus the contract layer, released 2026-09-16 to + # environments/base/python-cpu. # One image, so every variant pins the same digest until a variant # needs a base of its own. The prior digest, jupyter-python:0.2.1, # carried 31 fixable-critical findings the scan (E1-08) blocks every @@ -125,10 +126,10 @@ def repository(self) -> str: # `.spec.VARIANTS`, spelled out: `spec` imports from this # module, so importing it back here would be circular. ("datalayer", "e2b", "daytona", "modal"), - "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36", + "sha256:aa5413000bb5b6ecd0a0cf03959b107f0d572f65bf230c08bbdf9a4569775545", ) }, - snapshots={"2026.09": "20260914T150000Z"}, + snapshots={"2026.09": "20260916T120000Z"}, ), # E2-17: jupyter-python-cuda plus the same layer. ApprovedBase( From ab2dc58ef3d3e655d572f4dd9c66be8ce75629ae Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 12:17:37 +0200 Subject: [PATCH 07/22] release 1.9.15: a restart restarts the kernel, and kernels start in the contract's workdir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both found by the r1 geospatial drill on 2026-09-16 — the first one whose smoke test ran the Appendix B core tier for real against a built artifact. Seven of nine checks passed; these are the two that did not. - check 7 read "state survived the restart ('True')". CodeSandboxClient.restart() was stop() + start(), which destroys and recreates a sandbox this process owns but only drops the websocket of one attached to somebody else's Jupyter server — the kernel process keeps running and the reconnect lands in the same interpreter. JupyterServerSandbox.restart_kernel() now asks the server's own POST /api/kernels/{id}/restart, as _do_interrupt already does. - check 2 read "cwd is '/home/datalayer', not '/home/datalayer/content'". The image declares WORKDIR there and the contract's User row requires it, but a kernel's cwd is the Jupyter server's to choose and jupyter-python roots it at $HOME. The contract layer now sets MappingKernelManager.root_dir, which moves the kernel without moving the file browser. Channel repinned to sha256:122d3e31f5e2.... Also re-pinned the channel digest and apt snapshot in one place: they were duplicated across two test files and three base releases had left both red rather than catching anything. 1049 environment/client/jupyter-server tests pass; pre-commit clean. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 32 ++++++++++ code_sandboxes/__version__.py | 2 +- code_sandboxes/client.py | 16 ++++- code_sandboxes/environments/bases.py | 11 +++- code_sandboxes/jupyter_server_sandbox.py | 52 +++++++++++++++++ tests/test_client.py | 56 ++++++++++++++++++ tests/test_environment_bases.py | 21 +++++-- tests/test_environment_resolve.py | 6 +- tests/test_jupyter_server.py | 74 ++++++++++++++++++++++++ 9 files changed, 261 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de89de0..fbe8e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,38 @@ ## Unreleased +## 1.9.15 + +- **A restart restarts the kernel, not just this client's socket** + (`jupyter_server_sandbox`, `client`; PLAN_ENV.md E0-09, Appendix B check + 7). `CodeSandboxClient.restart()` was `stop()` then `start()`, which is + right for a sandbox this process owns — it is destroyed and recreated, and + nothing survives — and wrong for one *attached* to a Jupyter server + somebody else runs, which is every Datalayer runtime pod: stopping drops + the websocket while the kernel process keeps running, so the reconnect + lands in the same interpreter with every global still set. Check 7 is + "nothing is assumed to persist across restarts", and it read `state survived the restart ('True')` for exactly this reason — found live on r1, + 2026-09-16, the first drill whose smoke test reached the check. + `JupyterServerSandbox.restart_kernel()` now asks the server's own + `POST /api/kernels/{id}/restart` (the way `_do_interrupt` already uses the + API rather than the client's lifecycle) and reconnects onto the new + kernel; `restart()` prefers it and falls back to the lifecycle for every + variant that draws no such distinction. 7 new tests. +- **`datalayer/python-cpu:2026.09` repinned** to + `sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148`: + the contract layer now sets `MappingKernelManager.root_dir`, so kernels + start in `/home/datalayer/content`. The image already declared `WORKDIR` + there and `sandbox-contract/v1`'s User row already required it, but a + kernel's cwd is the Jupyter server's to choose and jupyter-python's config + roots it at `$HOME` — so every environment's kernel ran in + `/home/datalayer` and Appendix B check 2 read `cwd is '/home/datalayer', not '/home/datalayer/content'`. The file browser stays rooted at `$HOME`, + where a person expects to see everything they have; only the kernel moves. +- Two assertions that had rotted through three base releases are pinned in + one place again: the channel's digest and its apt snapshot were duplicated + across `test_environment_bases.py` and `test_environment_resolve.py`, and + 2026-09-15's and 2026-09-16's releases left both red rather than catching + anything. + ## 1.9.14 - **`datalayer/python-cpu:2026.09` base channel repinned** to the rebuilt diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 193120b..b93a62b 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.14" +__version__ = "1.9.15" diff --git a/code_sandboxes/client.py b/code_sandboxes/client.py index 3e8105f..4ac2604 100644 --- a/code_sandboxes/client.py +++ b/code_sandboxes/client.py @@ -508,7 +508,21 @@ def is_alive(self) -> bool: return self.is_started def restart(self) -> None: - """Restart the wrapped sandbox through its public lifecycle.""" + """Restart the wrapped sandbox, clearing what it was holding. + + A sandbox this process owns is restarted by its own lifecycle: stop + and start destroy and recreate it, and nothing survives. One that is + merely *attached* to a server somebody else runs — a Jupyter server + in a Datalayer runtime pod — is not: stopping drops this client's + websocket while the kernel process goes on running, and starting + reconnects to the same interpreter with every global still set. A + sandbox that knows how to restart what it is attached to says so with + `restart_kernel`, and that is used in preference; the lifecycle is + the fallback for every variant that has no such distinction. + """ + restart_kernel = getattr(self._sandbox, "restart_kernel", None) + if callable(restart_kernel) and restart_kernel(): + return self._sandbox.stop() self._sandbox.start() diff --git a/code_sandboxes/environments/bases.py b/code_sandboxes/environments/bases.py index 41f4669..a374e7c 100644 --- a/code_sandboxes/environments/bases.py +++ b/code_sandboxes/environments/bases.py @@ -126,7 +126,16 @@ def repository(self) -> str: # `.spec.VARIANTS`, spelled out: `spec` imports from this # module, so importing it back here would be circular. ("datalayer", "e2b", "daytona", "modal"), - "sha256:aa5413000bb5b6ecd0a0cf03959b107f0d572f65bf230c08bbdf9a4569775545", + # 2026-09-16: kernels now start in the contract's own + # working directory. The image already declared `WORKDIR + # /home/datalayer/content`, but a kernel's cwd is the + # Jupyter server's to choose, and jupyter-python's config + # roots it at `$HOME` — so every environment's kernel ran + # in `/home/datalayer` and Appendix B check 2 read "cwd is + # '/home/datalayer', not '/home/datalayer/content'". The + # contract layer now sets `MappingKernelManager.root_dir`, + # which moves the kernel without moving the file browser. + "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148", ) }, snapshots={"2026.09": "20260916T120000Z"}, diff --git a/code_sandboxes/jupyter_server_sandbox.py b/code_sandboxes/jupyter_server_sandbox.py index cbb0aec..a729f69 100644 --- a/code_sandboxes/jupyter_server_sandbox.py +++ b/code_sandboxes/jupyter_server_sandbox.py @@ -607,6 +607,58 @@ def _do_interrupt(self) -> bool: logger.warning(f"Failed to interrupt Jupyter kernel: {e}") return False + def restart_kernel(self) -> bool: + """Restart the kernel itself, through the server's own REST API. + + Not `stop()` then `start()`: those are this *client's* lifecycle, and + when the server is somebody else's — a Datalayer runtime pod, which + is every attached sandbox — stopping the client only drops the + websocket. The kernel is a process on the server and keeps running, + so a reconnect lands back in the same interpreter with every global + still set. + + That is what Appendix B check 7 ("nothing is assumed to persist + across restarts") measures, and it read `state survived the restart` + for exactly this reason — found live on r1, 2026-09-16, the first + drill whose smoke test reached the check. `POST + /api/kernels/{id}/restart` is the one that restarts the kernel, the + same way `_do_interrupt` already uses the API rather than the + client's own lifecycle. + + Answers whether the server accepted it; never raises, so a caller + that cannot restart reports a failed check rather than an error. + """ + if not self._server_url or not self._client: + return False + kernel_id = getattr(self._client, "id", None) + if not kernel_id: + return False + try: + response = requests.post( + f"{self._server_url}/api/kernels/{kernel_id}/restart", + params={"token": self._token}, + headers=self._headers or None, + timeout=30, + ) + except Exception as error: + # A restart that cannot even be asked for is a failed check, not + # an error to raise at the caller, exactly as `_do_interrupt` is. + logger.warning(f"Failed to restart Jupyter kernel: {error}") + return False + if not response.ok: + logger.warning( + "Failed to restart Jupyter kernel: the server answered %s", response.status_code + ) + return False + # The websocket the client holds is to the kernel that has just been + # replaced; reconnecting is what makes the next execution land in the + # new interpreter rather than on a channel nobody is reading. + with contextlib.suppress(Exception): + self._client.stop() + with contextlib.suppress(Exception): + self._client.start() + return True + @marks_execution def run_code( # noqa: C901 self, diff --git a/tests/test_client.py b/tests/test_client.py index c75c86e..34df743 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -157,6 +157,62 @@ def tool_caller(): assert client.is_alive() is True +class TestRestartingWhatIsActuallyHoldingTheState: + """A restart has to clear the interpreter, not just this client's socket. + + Appendix B check 7 is "nothing is assumed to persist across restarts". A + sandbox this process owns is cleared by its own lifecycle; one attached + to somebody else's Jupyter server is not, because the kernel outlives + the websocket. Found live on r1, 2026-09-16: `state survived the + restart`. + """ + + def test_a_sandbox_that_can_restart_its_kernel_is_asked_to(self): + class _AttachedSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.kernel_restarts = 0 + + def restart_kernel(self): + self.kernel_restarts += 1 + return True + + sandbox = _AttachedSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.kernel_restarts == 1 + # And the lifecycle was left alone: stopping it would have dropped a + # connection the restarted kernel is still reachable on. + assert sandbox.is_started is True + + def test_a_kernel_restart_that_fails_falls_back_to_the_lifecycle(self): + class _RefusingSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.stops = 0 + + def restart_kernel(self): + return False + + def stop(self): + self.stops += 1 + super().stop() + + sandbox = _RefusingSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.stops == 1, "a refused kernel restart still restarts the sandbox" + assert sandbox.is_started is True + + def test_a_sandbox_with_no_kernel_of_its_own_restarts_as_it_always_did(self): + sandbox = _FakeSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + assert sandbox.is_started is True + + def test_a_sandbox_that_knows_it_is_gone_is_believed(): """`is_alive` reports what the sandbox can find out, not the local start flag, so a backend that died under us is not reported as ready.""" diff --git a/tests/test_environment_bases.py b/tests/test_environment_bases.py index c2a185a..0de2f97 100644 --- a/tests/test_environment_bases.py +++ b/tests/test_environment_bases.py @@ -52,10 +52,17 @@ def test_the_2026_09_channel_of_python_cuda_has_no_digest_until_it_is_pushed( def test_the_2026_09_channel_of_python_cpu_resolves_the_digest_its_release_pushed( variant: str, ) -> None: - """PLAN_ENV.md, E1-05: released 2026-09-14 (jupyter-python 0.2.2, E1-08's - scan fix), same digest for every variant.""" + """PLAN_ENV.md, E1-05: the digest the channel's last release pushed, the + same one for every variant. + + **This moves with every base release**, and `bases.py` is where it moves + first: two releases (2026-09-15's and 2026-09-16's) changed the channel + and left this assertion on 2026-09-14's digest, so it sat red rather than + catching anything. Current: released 2026-09-16, the contract layer that + starts kernels in `/home/datalayer/content` (E1-05, Appendix B check 2). + """ ref = "datalayer/python-cpu" - digest = "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36" + digest = "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148" assert APPROVED_BASES[ref].channels == {"2026.09": dict.fromkeys(VARIANTS, digest)} assert resolve_base(ref, "2026.09", variant) == digest @@ -114,8 +121,12 @@ def test_each_base_is_published_under_its_own_repository() -> None: def test_the_2026_09_channel_of_python_cpu_pins_apt_to_its_snapshot() -> None: - """D-9: the moment just after the channel's image upgraded its packages.""" - assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260914T150000Z" + """D-9: the moment just after the channel's image upgraded its packages. + + Moves with the channel, like the digest above, and had rotted the same + way — left on 2026-09-14's id after the channel moved to 2026-09-16's. + """ + assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260916T120000Z" assert channel_snapshot("datalayer/python-cuda", "2026.09") == "" assert channel_snapshot("datalayer/nothing", "2026.09") == "" diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 72e308d..85447c6 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -998,8 +998,12 @@ def test_the_solve_is_asked_to_pin_apt_at_the_base_channels_snapshot() -> None: runner = RecordedRunner(A_LOCK) resolve_environment(spec=a_spec(), variants=["datalayer"], runner=runner) assert runner.request is not None + # What this test is about: the solve is pinned at *the channel's* snapshot, + # whatever that is. The id itself is pinned once, in + # `test_environment_bases.py`, where it moves with the channel — repeating + # it here only meant a base release left two tests red instead of one. assert runner.request.apt_snapshot == channel_snapshot("datalayer/python-cpu", "2026.09") - assert runner.request.apt_snapshot == "20260914T150000Z" + assert runner.request.apt_snapshot class TestTheBuildkitRunner: diff --git a/tests/test_jupyter_server.py b/tests/test_jupyter_server.py index 9c0be8c..4de3b51 100644 --- a/tests/test_jupyter_server.py +++ b/tests/test_jupyter_server.py @@ -526,3 +526,77 @@ def test_stop_forgets_the_temporary_workdir_it_removed(tmp_path: Path, monkeypat second = Path(sandbox._resolve_workdir()) assert second.is_dir() assert second != first + + +class TestRestartingTheKernelRatherThanTheConnection: + """Appendix B check 7, against a server this process does not own. + + `stop()` then `start()` is this client's lifecycle: when the Jupyter + server belongs to somebody else — a Datalayer runtime pod, which is every + attached sandbox — it drops the websocket and leaves the kernel process + running, so the next execution lands in the same interpreter with every + global still set. Found live on r1, 2026-09-16: the smoke test read + `state survived the restart`. + """ + + def test_the_kernel_is_restarted_through_the_server_s_own_api(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + asked: dict = {} + + class _Response: + ok = True + status_code = 200 + + def _post(url, params=None, headers=None, timeout=None): + asked["url"] = url + asked["params"] = params + return _Response() + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _post) + try: + assert sandbox.restart_kernel() is True + assert asked["url"].endswith("/api/kernels/kernel-1/restart") + finally: + sandbox.stop() + + def test_a_server_that_refuses_the_restart_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + class _Response: + ok = False + status_code = 503 + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", + lambda *args, **kwargs: _Response(), + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_server_that_cannot_be_reached_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + def _explode(*args, **kwargs): + raise OSError("no route to host") + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _explode) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_sandbox_with_no_kernel_id_asks_nothing(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id=None) + + def _should_not_be_called(*args, **kwargs): + raise AssertionError("the server must not be asked without a kernel id") + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", _should_not_be_called + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() From 121e7cb565011bc00c0594790829f61394fcabee Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 13:56:46 +0200 Subject: [PATCH 08/22] content --- code_sandboxes/environments/files.py | 24 +++++++++---- code_sandboxes/environments/spec.py | 16 +++++++++ schemas/environment-v1alpha1.json | 46 ++++++++++++++++++++++++ tests/test_environment_builders.py | 52 ++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 6 deletions(-) diff --git a/code_sandboxes/environments/files.py b/code_sandboxes/environments/files.py index e31cac6..e26f516 100644 --- a/code_sandboxes/environments/files.py +++ b/code_sandboxes/environments/files.py @@ -25,12 +25,15 @@ def build_entries( environment: Environment, *, source_of: Callable[[FileEntry], str] | None = None ) -> list[BuildEntry]: - """The spec's files as build entries. + """The spec's baked files as build entries: ``files`` then ``contentsBuild``. - ``source_of`` turns a ``contentRef`` into a URL the build can fetch — - a presigned URL for a ``blob://`` reference — and defaults to the - reference itself. A file without its sha256 is refused: the build - verifies every byte it bakes. + ``files`` a user uploaded (referenced by ``contentRef``) and + ``contentsBuild`` fetched from an external ``source`` are both baked the + same way, so they are the same kind of entry here. ``source_of`` turns a + ``contentRef`` into a URL the build can fetch — a presigned URL for a + ``blob://`` reference — and defaults to the reference itself; a + ``contentsBuild`` source is already a URL and is used as is. A file without + its sha256 is refused: the build verifies every byte it bakes. """ entries: list[BuildEntry] = [] for index, entry in enumerate(environment.spec.files): @@ -48,6 +51,15 @@ def build_entries( size_bytes=entry.size_bytes, ) ) + for built in environment.spec.contents_build: + entries.append( + BuildEntry( + source_uri=built.source, + destination_path=built.path, + sha256=built.sha256, + size_bytes=built.size_bytes, + ) + ) return entries @@ -63,7 +75,7 @@ def files_step( """ if variant not in VARIANTS: raise ValueError(f"{variant!r} is not a variant") - if not environment.spec.files: + if not environment.spec.files and not environment.spec.contents_build: return [] build = EnvironmentBuild( environment=environment.metadata.name, diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index 5d064b8..d180f9a 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -259,6 +259,21 @@ class FileEntry(_Model): sha256: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class ContentsBuildEntry(_Model): + """One immutable file the Environment bakes from an external source. + + Unlike ``files`` — which a user uploads, referenced by ``contentRef`` — a + ``contentsBuild`` entry names an external ``source`` URL fetched at build + time and verified against ``sha256`` (required: the build verifies every + byte it bakes). Both are baked the same way, into every provider artifact. + """ + + source: str + path: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size_bytes: int | None = Field(default=None, ge=0) + + class Commands(_Model): post_install: list[str] = Field(default_factory=list) @@ -365,6 +380,7 @@ class EnvironmentSpec(_Model): platform: Platform = Field(default_factory=Platform) packages: Packages = Field(default_factory=Packages) files: list[FileEntry] = Field(default_factory=list) + contents_build: list[ContentsBuildEntry] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) commands: Commands = Field(default_factory=Commands) build_secrets: list[BuildSecret] = Field(default_factory=list) diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index c6bdab7..2c44291 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -152,6 +152,45 @@ "title": "Compatibility", "type": "object" }, + "ContentsBuildEntry": { + "additionalProperties": false, + "description": "One immutable file the Environment bakes from an external source.\n\nUnlike ``files`` \u2014 which a user uploads, referenced by ``contentRef`` \u2014 a\n``contentsBuild`` entry names an external ``source`` URL fetched at build\ntime and verified against ``sha256`` (required: the build verifies every\nbyte it bakes). Both are baked the same way, into every provider artifact.", + "properties": { + "path": { + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "sizeBytes": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sizebytes" + }, + "source": { + "title": "Source", + "type": "string" + } + }, + "required": [ + "source", + "path", + "sha256" + ], + "title": "ContentsBuildEntry", + "type": "object" + }, "DependencyFileSpec": { "additionalProperties": false, "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.", @@ -202,6 +241,13 @@ "compatibility": { "$ref": "#/$defs/Compatibility" }, + "contentsBuild": { + "items": { + "$ref": "#/$defs/ContentsBuildEntry" + }, + "title": "Contentsbuild", + "type": "array" + }, "contract": { "default": "sandbox-contract/v1", "title": "Contract", diff --git a/tests/test_environment_builders.py b/tests/test_environment_builders.py index 4596ecc..28eb4fb 100644 --- a/tests/test_environment_builders.py +++ b/tests/test_environment_builders.py @@ -280,6 +280,58 @@ def test_a_file_is_not_baked_without_its_digest() -> None: files_step(environment(), variant="kaggle") +CONTENTS_SHA = "c" * 64 + + +def test_the_files_step_bakes_a_contents_build_manifest_from_an_external_source() -> None: + env = environment( + files=[], + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ], + ) + commands = files_step(env, variant="modal") + # The external source is fetched as it is (no contentRef to sign) and verified. + assert ( + "curl -fsSL https://data.example/iris.csv -o /opt/datalayer/contents/iris.csv" + in commands[0] + ) + assert f'echo "{CONTENTS_SHA} /opt/datalayer/contents/iris.csv" | sha256sum -c' in commands[0] + assert "environment-contents.json" in commands[-1] + + +def test_files_and_contents_build_are_baked_together() -> None: + env = environment( + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ] + ) + entries = build_entries(env, source_of=lambda entry: "https://signed.example/notes.md") + # The uploaded file first, then the external build entry — both baked, one engine. + assert [entry.destination_path for entry in entries] == [ + "/home/datalayer/content/notes.md", + "/opt/datalayer/contents/iris.csv", + ] + assert entries[0].source_uri == "https://signed.example/notes.md" + assert entries[1].source_uri == "https://data.example/iris.csv" + assert entries[1].sha256 == CONTENTS_SHA + + +def test_a_contents_build_entry_without_its_digest_will_not_parse() -> None: + with pytest.raises(Exception): + environment( + contents_build=[{"source": "https://data.example/x", "path": "/opt/x"}] + ) + + def test_the_neutral_modules_import_no_provider_sdk() -> None: """What a service may import must not drag a provider in.""" code = ( From d023b4eea8e46feb3adfa8077a7a1bac64117a59 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 14:17:56 +0200 Subject: [PATCH 09/22] release 1.9.16: bake an Environment's contents_build manifest via the shared engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit contents_build is a first-class field on the environment build spec, baked through the same build_commands engine as the spec's uploaded files (build_entries/files_step) on every provider adapter — verified fetch, checksum that fails the build, environment-contents.json manifest. Co-Authored-By: Claude Opus 4.8 --- code_sandboxes/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index b93a62b..e2255f9 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.15" +__version__ = "1.9.16" From 398ca274b1f3b6e57d624b9b50226b20e937b97d Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 15:38:52 +0200 Subject: [PATCH 10/22] release 1.9.17: an artifact's size is read from the registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `attest_artifact` took `size_bytes` from its caller and nobody ever passed one — the builder answers a reference, not a weight — so every artifact was recorded with `sizeBytes: null`, and `environments.artifact.bytes`, the series section 14 tracks the artifact size in, had no point in it although artifacts had been recorded. Seen on r1 on 2026-09-16, reading the section 14 SLOs back through the OTEL query API (PLAN_ENVS.md E1-25). `Attestor.size_of()` asks the registry with the client the scan is already read from. A size that cannot be read is logged, not raised: a missing number on a dashboard is no reason to refuse an artifact that is otherwise signed. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 13 +++++++++ code_sandboxes/__version__.py | 2 +- code_sandboxes/environments/attest.py | 27 ++++++++++++++++- tests/test_environment_attest.py | 42 ++++++++++++++++++++++++++- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbe8e9c..f960ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ ## Unreleased +## 1.9.17 + +- **An artifact's size is read from the registry** (`environments/attest.py`; + PLAN_ENVS.md E1-25). `attest_artifact` took `size_bytes` from its caller and + nobody ever passed one — the builder answers a reference, not a weight — so + every artefact was recorded with `sizeBytes: null` and + `environments.artifact.bytes`, the series section 14 tracks the artifact + size in, had no point in it although artifacts had been recorded (seen on r1, + 2026-09-16, through the OTEL query API). `Attestor.size_of()` asks the + registry, with the client the scan is already read from, and a size that + cannot be read is logged rather than raised: a missing number on a dashboard + is not a reason to refuse an artifact that is otherwise signed. 3 new tests. + ## 1.9.15 - **A restart restarts the kernel, not just this client's socket** diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index e2255f9..9964ed6 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.16" +__version__ = "1.9.17" diff --git a/code_sandboxes/environments/attest.py b/code_sandboxes/environments/attest.py index 94c750d..fcf7d88 100644 --- a/code_sandboxes/environments/attest.py +++ b/code_sandboxes/environments/attest.py @@ -536,10 +536,35 @@ def attest( signature_ref=signature, sbom_ref=sbom_ref or f"{registry}/{repository}@{digest}.sbom", provenance_ref=provenance_ref or f"{registry}/{repository}@{digest}.att", - size_bytes=size_bytes, + size_bytes=size_bytes + if size_bytes is not None + else self.size_of(repository=repository, digest=digest), signed_now=signed_now, ) + def size_of(self, *, repository: str, digest: str) -> int | None: + """What the registry says the artifact weighs, or None. + + Nobody hands the size down: the builder answers a reference, not a + weight, so until this asked the registry `size_bytes` was always None + — and with it `environments.artifact.bytes`, the series section 14 + tracks the artifact size in, which had no point in it on 2026-09-16 + although artifacts had been recorded. The registry has known all + along; the scan is read from the same client. + + Never a reason to fail an attestation: a size that could not be read + is a missing number on a dashboard, and the artifact is still signed. + """ + try: + images = self._client().describe_images( + repositoryName=repository, imageIds=[{"imageDigest": digest}] + )["imageDetails"] + except Exception as error: + self._log(f"The artifact's size could not be read: {error}") + return None + size = (images[0] or {}).get("imageSizeInBytes") if images else None + return int(size) if size else None + def _client(self) -> Any: if self._ecr is None: try: diff --git a/tests/test_environment_attest.py b/tests/test_environment_attest.py index 90038b7..842f3c5 100644 --- a/tests/test_environment_attest.py +++ b/tests/test_environment_attest.py @@ -80,7 +80,7 @@ def basic(identifier: str, severity: str, *, package: str = "openssl") -> dict: class FakeEcr: - """The two ECR calls the attestor makes, and nothing else.""" + """The three ECR calls the attestor makes, and nothing else.""" def __init__( self, @@ -90,6 +90,7 @@ def __init__( enhanced_findings=True, manifest: dict | None = None, pages: list[list[dict]] | None = None, + size_bytes: int | None = 2_147_483_648, ) -> None: self.statuses = list(statuses) self.findings = list(findings) @@ -99,8 +100,12 @@ def __init__( self.pages = pages self.enhanced_findings = enhanced_findings self.manifest = manifest + #: What `describe_images` answers for the digest; None makes it raise + #: `ImageNotFoundException`, the way a registry that lost it would. + self.size_bytes = size_bytes self.asked = 0 self.scanned: list[str] = [] + self.sized: list[str] = [] def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa: N803 - boto3's spelling manifest = self.manifest or { @@ -117,6 +122,14 @@ def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa ] } + def describe_images(self, repositoryName, imageIds): # noqa: N803 - boto3's spelling + self.sized.append(imageIds[0]["imageDigest"]) + if self.size_bytes is None: + error = Exception("ImageNotFoundException") + error.response = {"Error": {"Code": "ImageNotFoundException"}} + raise error + return {"imageDetails": [{"imageSizeInBytes": self.size_bytes}]} + def describe_image_scan_findings(self, repositoryName, imageId, nextToken=None): # noqa: N803 - boto3's spelling self.asked += 1 self.scanned.append(imageId["imageDigest"]) @@ -630,6 +643,33 @@ def test_the_policy_it_was_decided_under_is_part_of_the_record(self) -> None: answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor()) assert answer["scan_summary"]["policy"] == DEFAULT_POLICY.body() + def test_the_size_is_read_from_the_registry_when_nobody_hands_one_down(self) -> None: + """Which is every real call: the builder answers a reference, not a + weight, so `environments.artifact.bytes` — section 14's artifact size + — had no point in it although artifacts had been recorded (E1-25).""" + ecr = FakeEcr(size_bytes=2_147_483_648) + answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor(ecr=ecr)) + assert answer["size_bytes"] == 2_147_483_648 + assert ecr.sized == [DIGEST] + + def test_a_size_handed_down_is_kept_and_the_registry_is_not_asked(self) -> None: + ecr = FakeEcr() + answer = attest_artifact( + artifact=self.an_artifact(), size_bytes=116_183_040, attestor=an_attestor(ecr=ecr) + ) + assert answer["size_bytes"] == 116_183_040 + assert ecr.sized == [] + + def test_a_size_that_cannot_be_read_is_not_a_reason_to_refuse(self) -> None: + """A missing number on a dashboard, against an artifact nothing can + launch: the artifact is signed and the attestation stands.""" + said: list[str] = [] + attestor = an_attestor(ecr=FakeEcr(size_bytes=None), log=said.append) + answer = attest_artifact(artifact=self.an_artifact(), attestor=attestor) + assert answer["size_bytes"] is None + assert answer["signature_ref"] == f"{REGISTRY}/{REPOSITORY}@{DIGEST}" + assert any("size could not be read" in line for line in said) + def test_nothing_reaches_a_registry_when_nothing_could_sign() -> None: """The order that keeps a refusal cheap and honest (E1-08, E1-09). From a7e4973957b8d3b3b7689c4aeb7c50c4173561be Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Wed, 16 Sep 2026 17:54:26 +0200 Subject: [PATCH 11/22] release 1.9.18: a lock no longer carries a wall clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lock's header carried `# resolved-at:`, and its digest is over the whole text, so two resolves of the same spec in the same base pinning the same 320 packages produced two different digests. Found by resolving one environment twice on r1: the texts differed in exactly that line, one of 5,388. Section 5's cache key is over the lock digest, so D-12's build cache could never hit — and it never had, `hit=false` twelve times out of twelve. When a lock was resolved is on the lock document Runtimes stores, in `created_at`. The two tests that asserted determinism did it by freezing the clock. They assert it without one now. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 17 ++++++++ code_sandboxes/__version__.py | 2 +- code_sandboxes/environments/resolve.py | 18 ++++---- code_sandboxes/environments/resolve_conda.py | 45 ++++++++++---------- tests/test_environment_resolve.py | 14 ++++-- tests/test_environment_resolve_conda.py | 24 ++++------- 6 files changed, 69 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f960ab0..5a417bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,23 @@ ## Unreleased +## 1.9.18 + +- **A lock no longer carries a wall clock, so the build cache can hit** + (`environments/resolve.py`, `resolve_conda.py`; PLAN_ENVS.md E1-26, D-12). + The lock's header carried a `# resolved-at:` line, and its digest is over + the whole text — so two resolves of the same spec, in the same base, + pinning the same 320 packages, produced two different digests. Found by + resolving one environment twice on r1 on 2026-09-16: the texts differed in + exactly that one line out of 5,388. Section 5's cache key is over the lock + digest, so D-12's build cache could never hit, and it never had: + `environments.cache.lookups` read `hit=false` twelve times out of twelve. + When a lock was resolved is on the lock document Runtimes stores, in its + `created_at`, which is where it belongs. `resolved_at` is gone from + `lock_document`, `conda_lock_document`, `resolve_environment` and + `resolve_conda_environment`; the two tests that asserted determinism by + freezing the clock now assert it without one. + ## 1.9.17 - **An artifact's size is read from the registry** (`environments/attest.py`; diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 9964ed6..ed7b561 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.17" +__version__ = "1.9.18" diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 5a9fed6..16b9f04 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -889,7 +889,6 @@ def lock_document( python_version: str, base_reference: str, merged: MergedRequirements, - resolved_at: datetime | None = None, ) -> dict[str, Any]: """The stored lock: its text, its digest, and what a reader needs from it. @@ -897,11 +896,19 @@ def lock_document( output, so the document is still a requirements file — ``pip install -r`` reads it, and so does every tool that only knows that format — while saying everything the build installs. + + **Nothing here is a wall clock.** The header carried a ``# resolved-at:`` + line until 2026-09-16, and since the digest is over the whole text, two + resolves of the same spec, in the same base, pinning the same 320 packages + produced two different digests — the texts differed in that one line out + of 5,388, found by resolving the same environment twice on r1. The cache + key of section 5 is over the lock digest, so D-12's build cache could + never hit: `environments.cache.lookups` read `hit=false` 12 times out of + 12. When the lock was resolved is on the lock document Runtimes stores, in + its ``created_at``, where it belongs. """ - when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() header = [ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", - f"# resolved-at: {when}", f"# python: {python_version}", f"# base: {base_reference}", ] @@ -1163,7 +1170,6 @@ def resolve_environment( runner: ResolveRunner | None = None, conda_runner: CondaResolveRunner | None = None, bases: dict[str, ApprovedBase] = APPROVED_BASES, - resolved_at: datetime | None = None, uv: str | None = None, pyproject_run: Callable[..., subprocess.CompletedProcess[str]] | None = None, image_transport: Any = None, @@ -1198,8 +1204,6 @@ def resolve_environment( bases The approved bases, injected by tests and by a plane whose channel is published somewhere else. - resolved_at - The moment the lock records. Now by default. uv, pyproject_run A `pyproject` `dependencyFile` source's own verification (E3-01): the `uv` to check and export with, and how it is run — injected by @@ -1274,7 +1278,6 @@ def resolve_environment( credential=credential, log=say, runner=conda_runner, - resolved_at=resolved_at, ) if dependency_file.source_format == "pyproject": # Verified, not re-resolved (E3-01): the author's own uv.lock is @@ -1321,7 +1324,6 @@ def resolve_environment( python_version=environment.spec.language.version, base_reference=solving_in, merged=merged, - resolved_at=resolved_at, ) say( f"Locked {document['package_count']} packages" diff --git a/code_sandboxes/environments/resolve_conda.py b/code_sandboxes/environments/resolve_conda.py index 8a7fc94..e6462dd 100644 --- a/code_sandboxes/environments/resolve_conda.py +++ b/code_sandboxes/environments/resolve_conda.py @@ -644,24 +644,28 @@ def dockerfile(self, request: CondaResolveRequest) -> str: carries the whole of what the solve resolved, not the conda layer alone. """ micromamba = shlex.quote(MICROMAMBA_BINARY) - return "\n".join( - [ - f"FROM {request.base_reference} AS solve", - "USER root", - "WORKDIR /solve", - micromamba_bootstrap_dockerfile_line(), - "COPY environment.yml ./environment.yml", - "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), - "RUN --mount=type=cache,target=/opt/conda/pkgs " - f"{micromamba} create --yes --prefix /solve/prefix " - f"--platform {shlex.quote(request.platform)} --file environment.yml", - f"RUN {micromamba} env export --explicit --prefix /solve/prefix > /solve/lock.txt", - f"RUN {micromamba} env export --prefix /solve/prefix > /solve/pip-env.yml", - "FROM scratch", - "COPY --from=solve /solve/lock.txt /lock.txt", - "COPY --from=solve /solve/pip-env.yml /pip-env.yml", - ] - ) + "\n" + return ( + "\n".join( + [ + f"FROM {request.base_reference} AS solve", + "USER root", + "WORKDIR /solve", + micromamba_bootstrap_dockerfile_line(), + "COPY environment.yml ./environment.yml", + "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), + "RUN --mount=type=cache,target=/opt/conda/pkgs " + f"{micromamba} create --yes --prefix /solve/prefix " + f"--platform {shlex.quote(request.platform)} --file environment.yml", + f"RUN {micromamba} env export --explicit " + "--prefix /solve/prefix > /solve/lock.txt", + f"RUN {micromamba} env export --prefix /solve/prefix > /solve/pip-env.yml", + "FROM scratch", + "COPY --from=solve /solve/lock.txt /lock.txt", + "COPY --from=solve /solve/pip-env.yml /pip-env.yml", + ] + ) + + "\n" + ) def solve( self, request: CondaResolveRequest, log: Callable[[str], None] | None = None @@ -800,7 +804,6 @@ def conda_lock_document( base_reference: str, merged: MergedRequirements, platform: str = CONDA_PLATFORM, - resolved_at: datetime | None = None, ) -> dict[str, Any]: """The stored conda lock: its text, its digest, and what a reader needs. @@ -814,10 +817,8 @@ def conda_lock_document( could read the prefix back, and the merged requirements otherwise, so it is always complete rather than the protected pins alone. """ - when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() header = [ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", - f"# resolved-at: {when}", f"# python: {python_version}", f"# platform: {platform}", f"# base: {base_reference}", @@ -854,7 +855,6 @@ def resolve_conda_environment( credential: Any = None, log: Callable[[str], None] | None = None, runner: CondaResolveRunner | None = None, - resolved_at: datetime | None = None, ) -> dict[str, Any]: """A conda version's lock, from its ``environment.yml`` (E3-02). @@ -892,7 +892,6 @@ def resolve_conda_environment( base_reference=solving_in, merged=merged, platform=platform, - resolved_at=resolved_at, ) say(f"Locked {document['package_count']} conda packages as {document['digest']}") return {**document, "resolved_bases": dict(resolved_bases)} diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 85447c6..edeb79e 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -13,7 +13,6 @@ from __future__ import annotations import subprocess -from datetime import datetime, timezone import httpx import pytest @@ -266,7 +265,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) python_version="3.13", base_reference="environments/base/python-cpu@sha256:" + "11" * 32, merged=merge_requirements(["geopandas==1.1.1"]), - resolved_at=datetime(2026, 9, 12, 8, 30, tzinfo=timezone.utc), ) assert document["format"] == LOCK_FORMAT assert document["digest"].startswith("sha256:") @@ -275,7 +273,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) content = document["content"] assert f"{APT_PIN_PREFIX}gdal-bin=3.8.4+dfsg-3build2" in content assert "# datalayer-protected: ipykernel==7.3.0" in content - assert "# resolved-at: 2026-09-12T08:30:00+00:00" in content # The snapshot the pins came from, which the builder installs from. from code_sandboxes.environments.resolve import apt_pins_in, apt_snapshot_in @@ -289,15 +286,24 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) } def test_the_same_lock_digests_the_same_and_a_changed_one_does_not(self) -> None: + """And nothing here may pin a clock to make it true. + + The header carried a `# resolved-at:` line until 2026-09-16, and this + test passed only because it froze the moment. Resolving the same spec + twice on r1 produced two digests differing in that one line out of + 5,388, and section 5's cache key is over the lock digest — so D-12's + build cache had never hit, 12 lookups out of 12. + """ arguments = { "python_version": "3.13", "base_reference": "environments/base/python-cpu@sha256:" + "11" * 32, "merged": MergedRequirements((), (), ()), - "resolved_at": datetime(2026, 9, 12, tzinfo=timezone.utc), } first = lock_document(A_LOCK, **arguments) again = lock_document(A_LOCK, **arguments) assert first["digest"] == again["digest"] + assert first["content"] == again["content"] + assert "resolved-at" not in first["content"] moved = lock_document( ResolveOutcome(lock_text=A_LOCK.lock_text.replace("1.1.1", "1.1.2")), **arguments ) diff --git a/tests/test_environment_resolve_conda.py b/tests/test_environment_resolve_conda.py index fa4d414..26b3049 100644 --- a/tests/test_environment_resolve_conda.py +++ b/tests/test_environment_resolve_conda.py @@ -12,7 +12,6 @@ from __future__ import annotations import subprocess -from datetime import datetime, timezone import pytest @@ -192,8 +191,7 @@ def test_a_channel_url_carrying_a_credential_is_refused(self) -> None: def test_a_plain_channel_url_without_a_credential_is_kept(self) -> None: env = parse_conda_environment( - "channels:\n - https://conda.anaconda.org/conda-forge\n" - "dependencies:\n - gdal\n" + "channels:\n - https://conda.anaconda.org/conda-forge\n" "dependencies:\n - gdal\n" ) assert env.channels == ("https://conda.anaconda.org/conda-forge",) @@ -235,17 +233,13 @@ def test_the_protected_pins_are_forced_into_the_pip_layer(self) -> None: assert {"shapely", "ipykernel", "jupyter-server"} <= names def test_a_pip_requirement_contradicting_a_pin_is_refused(self) -> None: - env = parse_conda_environment( - "dependencies:\n - pip:\n - ipykernel==6.0.0\n" - ) + env = parse_conda_environment("dependencies:\n - pip:\n - ipykernel==6.0.0\n") with pytest.raises(EnvironmentsError) as caught: merge_conda_pip(env) assert caught.value.code.code == "DL_ENV_PROTECTED_PACKAGE" def test_the_interpreter_is_pinned_and_never_doubled(self) -> None: - env = parse_conda_environment( - "dependencies:\n - python=3.11\n - gdal=3.9\n" - ) + env = parse_conda_environment("dependencies:\n - python=3.11\n - gdal=3.9\n") merged = merge_conda_pip(env) rendered = rendered_environment(env, merged, python_version="3.13") assert rendered.count("python=3.13") == 1 @@ -291,13 +285,11 @@ def test_it_counts_only_the_package_urls(self) -> None: def test_the_document_records_the_pins_and_is_deterministic(self) -> None: env = parse_conda_environment(A_YAML) merged = merge_conda_pip(env) - at = datetime(2026, 9, 14, tzinfo=timezone.utc) document = conda_lock_document( CondaResolveOutcome(lock_text=EXPLICIT_LOCK), python_version="3.13", base_reference="registry/base@sha256:" + "11" * 32, merged=merged, - resolved_at=at, ) assert document["format"] == CONDA_LOCK_FORMAT assert document["package_count"] == 2 @@ -311,9 +303,13 @@ def test_the_document_records_the_pins_and_is_deterministic(self) -> None: python_version="3.13", base_reference="registry/base@sha256:" + "11" * 32, merged=merged, - resolved_at=at, ) + # Deterministic without anybody pinning a clock: the header carried a + # `# resolved-at:` line until 2026-09-16, which made two resolves of + # one spec two different digests and kept D-12's cache from ever + # hitting. assert document["digest"] == again["digest"] + assert "resolved-at" not in document["content"] def test_an_export_without_the_marker_is_a_provider_error(self) -> None: env = parse_conda_environment(A_YAML) @@ -353,9 +349,7 @@ def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[s return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") raise subprocess.TimeoutExpired(argv, 5.0) - monkeypatch.setattr( - "code_sandboxes.environments.resolve_conda.subprocess.run", fake_run - ) + monkeypatch.setattr("code_sandboxes.environments.resolve_conda.subprocess.run", fake_run) with pytest.raises(EnvironmentsError) as caught: runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" From 5741515aa52f675714fd32fc5cf3291cdf29e710 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 06:16:29 +0200 Subject: [PATCH 12/22] environments: read the licences an SBOM names (E2-16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A published version's page says it shows the licences its SBOM names, and every publication froze an empty list instead: the snapshot read them from the scan summary, and the registry's scanner reports vulnerabilities, not licences. There was nowhere for them to come from. `licenses_of` reads both shapes the ecosystem writes — SPDX, which is what BuildKit's `attest:sbom=` produces, and CycloneDX — taking a concluded licence over a declared one and treating SPDX's NOASSERTION as the non-answer it is. A document it does not understand names nothing rather than raising: a licence list is worth having and never worth failing a build over. `attest` takes the document and freezes what it found onto the artifact, so a publication carries it without reading anything at publish time. Still missing, and it needs registry access this machine does not have: the fetch of the SBOM itself. BuildKit pushes it as an OCI attestation in the image index, while `sbom_ref` is a constructed `…@digest.sbom` string that probably does not resolve, and the blob read is the Docker Registry HTTP API rather than boto3. Until a caller passes the document, the list stays empty — as it already was. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/environments/attest.py | 82 ++++++++++++++++++++++++++- tests/test_environment_attest.py | 62 +++++++++++++++++++- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/code_sandboxes/environments/attest.py b/code_sandboxes/environments/attest.py index fcf7d88..2db2d1c 100644 --- a/code_sandboxes/environments/attest.py +++ b/code_sandboxes/environments/attest.py @@ -106,6 +106,76 @@ def signature_tag(digest: str) -> str: return text.replace(":", "-", 1) + ".sig" +#: Where a CycloneDX component keeps its licence, in the order they are read. +_CYCLONEDX_LICENSE_KEYS = ("id", "name") +#: Where an SPDX package keeps its licence. `licenseConcluded` is what the +#: tool decided; `licenseDeclared` is what the package claimed. BuildKit +#: writes SPDX, so this is the one that matters in practice. +_SPDX_LICENSE_KEYS = ("licenseConcluded", "licenseDeclared") +#: What SPDX writes when it could not tell, which is not a licence. +_SPDX_UNKNOWN = frozenset({"NOASSERTION", "NONE", ""}) + + +def _spdx_licenses(document: Mapping[str, Any]) -> set[str]: + """What an SPDX document names, which is what BuildKit's `attest:sbom=` writes. + + `licenseConcluded` is what the tool decided and `licenseDeclared` what the + package claimed, so the concluded one is read first and the declared one + only when it said nothing. + """ + found: set[str] = set() + for package in document.get("packages") or (): + if not isinstance(package, Mapping): + continue + for key in _SPDX_LICENSE_KEYS: + value = str(package.get(key) or "").strip() + if value and value.upper() not in _SPDX_UNKNOWN: + found.add(value) + break + return found + + +def _cyclonedx_licenses(document: Mapping[str, Any]) -> set[str]: + """What a CycloneDX document names, by id, by name, or as an expression.""" + found: set[str] = set() + for component in document.get("components") or (): + if not isinstance(component, Mapping): + continue + for entry in component.get("licenses") or (): + if not isinstance(entry, Mapping): + continue + licence = entry.get("license") + if isinstance(licence, Mapping): + for key in _CYCLONEDX_LICENSE_KEYS: + value = str(licence.get(key) or "").strip() + if value: + found.add(value) + break + expression = str(entry.get("expression") or "").strip() + if expression: + found.add(expression) + return found + + +def licenses_of(document: Any) -> list[str]: + """Every licence an SBOM names, deduplicated and sorted. + + Reads both shapes the ecosystem writes: SPDX, which is what BuildKit's + `attest:sbom=` produces, and CycloneDX. A document in neither shape, or one + that names nothing, answers an empty list rather than raising: a + publication's licence list is worth having and never worth failing a build + over. + + This is what a published version's snapshot carries (D-12, E2-16). Until + it did, `licenses` came from the scan summary — and the registry's scanner + reports vulnerabilities, not licences, so every publication froze an empty + list beside an SBOM reference. + """ + if not isinstance(document, Mapping): + return [] + return sorted(_spdx_licenses(document) | _cyclonedx_licenses(document)) + + @dataclass(frozen=True) class AttestationResult: """What the workflow stores about an artifact once both gates have passed.""" @@ -117,6 +187,8 @@ class AttestationResult: size_bytes: int | None = None signed_now: bool = True """False when a replay found the signature that was already there.""" + licenses: tuple[str, ...] = () + """What the SBOM named, frozen onto the artifact for a publication to carry.""" def body(self) -> dict[str, Any]: """The mapping `activities_environments.attest` answers.""" @@ -127,6 +199,7 @@ def body(self) -> dict[str, Any]: "signature_ref": self.signature_ref, "size_bytes": self.size_bytes, "signed_now": self.signed_now, + "licenses": list(self.licenses), } @@ -515,8 +588,14 @@ def attest( size_bytes: int | None = None, sbom_ref: str = "", provenance_ref: str = "", + sbom: Any = None, ) -> AttestationResult: - """Scan, then sign: the order the Operator's check depends on (D-11).""" + """Scan, then sign: the order the Operator's check depends on (D-11). + + `sbom`, when the caller has the document, is read for the licences a + publication carries; the registry's scanner reports vulnerabilities + and never licences, so there is nowhere else they come from. + """ self.can_sign() decision = self.scan(repository=repository, digest=digest) if not decision.passed: @@ -540,6 +619,7 @@ def attest( if size_bytes is not None else self.size_of(repository=repository, digest=digest), signed_now=signed_now, + licenses=tuple(licenses_of(sbom)), ) def size_of(self, *, repository: str, digest: str) -> int | None: diff --git a/tests/test_environment_attest.py b/tests/test_environment_attest.py index 842f3c5..44ce658 100644 --- a/tests/test_environment_attest.py +++ b/tests/test_environment_attest.py @@ -22,7 +22,7 @@ import pytest -from code_sandboxes.environments.attest import Attestor, attest_artifact, signature_tag +from code_sandboxes.environments.attest import Attestor, attest_artifact, licenses_of, signature_tag from code_sandboxes.environments.builders import ArtifactReference from code_sandboxes.environments.errors import EnvironmentsError from code_sandboxes.environments.policy import ( @@ -689,3 +689,63 @@ def test_nothing_reaches_a_registry_when_nothing_could_sign() -> None: attest_artifact(artifact=artifact, attestor=an_attestor(ecr=ecr, key="")) assert raised.value.detail["missing"] == "DATALAYER_ENVIRONMENTS_KMS_KEY" assert ecr.asked == 0, "the registry was asked before anything could have been signed" + + +# -- the licences a publication carries --------------------------------------------------------- + + +class TestLicencesFromTheSbom: + """What `licenses_of` reads, and what it refuses to guess. + + A published version's page says it shows the licences its SBOM names + (D-12, E2-16). They had nowhere to come from: the snapshot read the scan + summary, and the registry's scanner reports vulnerabilities. + """ + + def test_it_reads_spdx_which_is_what_buildkit_writes(self) -> None: + document = { + "packages": [ + {"name": "gdal", "licenseConcluded": "MIT"}, + {"name": "numpy", "licenseConcluded": "BSD-3-Clause"}, + {"name": "again", "licenseConcluded": "MIT"}, + ] + } + assert licenses_of(document) == ["BSD-3-Clause", "MIT"] + + def test_a_concluded_licence_wins_over_a_declared_one(self) -> None: + """`licenseConcluded` is what the tool decided; `licenseDeclared` is the claim.""" + document = { + "packages": [ + {"licenseConcluded": "Apache-2.0", "licenseDeclared": "MIT"}, + ] + } + assert licenses_of(document) == ["Apache-2.0"] + + def test_noassertion_is_not_a_licence_and_falls_through(self) -> None: + """SPDX writes NOASSERTION when it could not tell, which must not be shown.""" + document = { + "packages": [ + {"licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause"}, + {"licenseConcluded": "NONE", "licenseDeclared": ""}, + ] + } + assert licenses_of(document) == ["BSD-3-Clause"] + + def test_it_reads_cyclonedx_by_id_by_name_and_by_expression(self) -> None: + document = { + "components": [ + {"licenses": [{"license": {"id": "Apache-2.0"}}]}, + {"licenses": [{"license": {"name": "Public Domain"}}]}, + {"licenses": [{"expression": "MIT OR Apache-2.0"}]}, + ] + } + assert licenses_of(document) == [ + "Apache-2.0", + "MIT OR Apache-2.0", + "Public Domain", + ] + + def test_a_document_it_does_not_understand_names_nothing(self) -> None: + """Never a reason to fail a build: a licence list is worth having, not dying for.""" + for document in (None, {}, {"packages": None}, {"components": [1, 2]}, "spdx"): + assert licenses_of(document) == [] From 5798478f8482e52087fa4b6a7a3a840f5f9a48e4 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 06:43:45 +0200 Subject: [PATCH 13/22] modal: record the layers a build leaves, and collect them (E2-05, E2-09) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each chained builder call leaves an image with an id of its own, and deleting the artifact does not delete them. Modal offers no call that lists an account's images — ImageGetOrCreate, ImageFromId, ImageGetByTag, ImageListTags, ImageTagRevisions, ImagePublish and ImageDelete, and nothing that enumerates — so an intermediate nobody writes down at build time can never be found again. That is why E2-05 says `delete` removes "the recorded intermediates", and it is why this records rather than discovers. Confirmed against a real Modal account, 2026-09-17: - `Image.build` hydrates an `object_id` on every image in `deps()`, not only on the last, so the whole chain is readable once the build finishes. - `ImageDelete` removes one, and the image is `NotFound` afterwards. - The bottom of a chain can be an image the workspace does not own: Modal's own `debian_slim` answers PermissionDenied. An image somebody else owns was never this artifact's to collect, so it is logged and stepped over. `ArtifactReference` gained `intermediates`; `build` records them; `delete` removes them before the artifact, since an intermediate is only reachable while the record naming it survives. Already-gone is success, the way a replayed collection has to be. 7 tests, and modal's `delete` leaves the not-built-yet list. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/environments/adapters/modal.py | 88 +++++++++- code_sandboxes/environments/builders.py | 8 + tests/test_environment_managed_builders.py | 8 +- tests/test_environment_modal_builder.py | 152 +++++++++++++++++- 4 files changed, 247 insertions(+), 9 deletions(-) diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index abf6f40..3c557ab 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -230,8 +230,7 @@ def _install_packages(image: Any, lock_text: str) -> Any: # Packages install as root: every Modal build step already runs as # root regardless of any `USER` line (see the module docstring), so # this is stating what is already true rather than asking for it. - "uv pip sync --system --require-hashes " - f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", + f"uv pip sync --system --require-hashes --find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", ) @@ -245,6 +244,40 @@ def _post_install( return image +def _intermediates_of(built: Any) -> tuple[str, ...]: + """Every layer under a built image, by id, deepest first (E2-05, E2-09). + + Each chained builder call leaves an image of its own, and deleting the + artifact does not delete them. Modal offers no call that lists an + account's images, so an intermediate nobody wrote down at build time can + never be found again — which is why this is recorded rather than + discovered. Confirmed live on 2026-09-17: a three-call chain built through + `Image.build` hydrates an `object_id` on every image in `deps()`, not only + on the last. + + The built image itself is not an intermediate: it is the artifact. + """ + found: list[str] = [] + seen: set[int] = set() + + def walk(image: Any) -> None: + if id(image) in seen: + return + seen.add(id(image)) + for dependency in getattr(image, "deps", lambda: ())(): + if hasattr(dependency, "deps"): + walk(dependency) + object_id = getattr(image, "object_id", None) + if object_id and image is not built: + found.append(str(object_id)) + + try: + walk(built) + except Exception: + return () + return tuple(dict.fromkeys(found)) + + class Builder(ManagedBuilder): """Modal: the capability half (E2-06) and the build (E2-05).""" @@ -479,6 +512,7 @@ def build(self, request: BuildRequest) -> ArtifactReference: mutable_alias=name, provider_account=provider_account(self.variant, self._provider_secrets()) or None, contract_version=spec.contract or SANDBOX_CONTRACT_V1.version, + intermediates=_intermediates_of(built), ) def _resolved_secrets(self, request: BuildRequest) -> tuple[list[BuildSecret], dict[str, str]]: @@ -631,6 +665,56 @@ def exists(self, artifact: ArtifactReference) -> bool: raise self._provider_error("ask whether the image exists", error) from error return True + def delete(self, artifact: ArtifactReference) -> None: + """Delete the image, and every intermediate layer this build recorded (E2-05, E2-09). + + Deleting the artifact does not delete the layers under it, and Modal + offers no call that lists an account's images, so what is removed is + what `build` wrote down — an intermediate nobody recorded can never be + found again. + + Two answers are outcomes rather than failures, both found live on + 2026-09-17: + + * **Already gone** is success. A replay of a collection must delete the + same set again with no harm, the same way the Datalayer collector + treats an artifact that is not there. + * **Not ours to delete.** The bottom of a chain can be an image the + workspace does not own — Modal's own `debian_slim` answers + `PermissionDenied` — and an image somebody else owns was never this + artifact's to collect. It is logged and stepped over, not raised. + """ + sdk = self._modal_sdk() + client = self._client(sdk) + synchronizer, api_pb2 = self._modal_internals() + + async def _delete(image_id: str) -> None: + await client.stub.ImageDelete(api_pb2.ImageDeleteRequest(image_id=image_id)) + + # The artifact last: an intermediate is only reachable while the + # record naming it survives, so a half-done collection that has + # dropped the image would strand them. + for image_id in (*artifact.intermediates, artifact.provider_artifact_id): + if not image_id: + continue + try: + # A bare coroutine on Modal's stub silently does nothing, so + # this runs on the SDK's own loop — see `_delete_secret`. + synchronizer.wrap(_delete)(image_id) + except sdk.exception.NotFoundError: + continue + except Exception as error: + if "permission" in str(error).lower(): + self._log( + f"The Modal image {image_id} is not this account's to delete: {error}" + ) + continue + if image_id == artifact.provider_artifact_id: + raise self._provider_error("delete the image", error) from error + # One layer's refusal does not strand the rest, nor the + # artifact this was called to collect. + self._log(f"The Modal intermediate {image_id} could not be deleted: {error}") + def _provider_error(self, what: str, error: BaseException) -> EnvironmentsError: return EnvironmentsError( PROVIDER_ERROR, diff --git a/code_sandboxes/environments/builders.py b/code_sandboxes/environments/builders.py index c0d4f34..9d786ee 100644 --- a/code_sandboxes/environments/builders.py +++ b/code_sandboxes/environments/builders.py @@ -141,6 +141,14 @@ class ArtifactReference(_Model): provider_account: str | None = None contract_version: str architecture: str = "linux/amd64" + #: The layers this build left behind that deleting the artifact does not + #: delete (E2-05, E2-09). Modal is the variant that has them: each chained + #: builder call leaves an image of its own with an id, and Modal offers no + #: call that lists an account's images — `ImageGetOrCreate`, `ImageFromId`, + #: `ImageGetByTag`, `ImageListTags`, `ImageTagRevisions`, `ImagePublish` + #: and `ImageDelete`, and nothing that enumerates — so an intermediate + #: nobody wrote down is an intermediate nobody can ever find again. + intermediates: tuple[str, ...] = () @model_validator(mode="after") def _immutable(self) -> ArtifactReference: diff --git a/tests/test_environment_managed_builders.py b/tests/test_environment_managed_builders.py index a473928..7816645 100644 --- a/tests/test_environment_managed_builders.py +++ b/tests/test_environment_managed_builders.py @@ -313,14 +313,14 @@ class TestTheHalfThatIsNotBuiltYet: `ManagedBuilder` methods on all three.""" def test_modal_still_refuses_what_e2_05_did_not_build(self) -> None: - """`build`/`inspect`/`exists` are E2-05's; `smoke_test`, `resolve` and - `delete` are not — see test_environment_modal_builder.py for what - is built.""" + """`build`/`inspect`/`exists` are E2-05's, and `delete` is now too — it + collects the intermediate layers a build recorded (E2-05, E2-09). + `smoke_test` and `resolve` are still not built — see + test_environment_modal_builder.py for what is.""" builder = get_builder("modal") calls = { "smoke_test": lambda: builder.smoke_test(None), # type: ignore[arg-type] "resolve": lambda: builder.resolve("geo@1"), - "delete": lambda: builder.delete(None), # type: ignore[arg-type] } for operation, call in calls.items(): with pytest.raises(EnvironmentsError) as raised: diff --git a/tests/test_environment_modal_builder.py b/tests/test_environment_modal_builder.py index 0adc96f..7093b9a 100644 --- a/tests/test_environment_modal_builder.py +++ b/tests/test_environment_modal_builder.py @@ -237,20 +237,41 @@ class FakeStub: `def`-ined, the same way `test_environment_daytona_builder.py` handles the same clash for `DockerRegistryApi`.""" - def __init__(self, *, delete_error: Exception | None = None) -> None: + def __init__( + self, + *, + delete_error: Exception | None = None, + image_delete_errors: dict[str, Exception] | None = None, + ) -> None: self.secret_delete_calls: list[Any] = [] self._delete_error = delete_error self.SecretDelete = self._secret_delete + #: Every image this stub was asked to delete, in order. + self.image_delete_calls: list[str] = [] + self._image_delete_errors = dict(image_delete_errors or {}) + self.ImageDelete = self._image_delete async def _secret_delete(self, request: Any) -> None: self.secret_delete_calls.append(request) if self._delete_error: raise self._delete_error + async def _image_delete(self, request: Any) -> None: + image_id = request.kwargs["image_id"] + self.image_delete_calls.append(image_id) + error = self._image_delete_errors.get(image_id) + if error: + raise error + class FakeClient: - def __init__(self, *, delete_error: Exception | None = None) -> None: - self.stub = FakeStub(delete_error=delete_error) + def __init__( + self, + *, + delete_error: Exception | None = None, + image_delete_errors: dict[str, Exception] | None = None, + ) -> None: + self.stub = FakeStub(delete_error=delete_error, image_delete_errors=image_delete_errors) class FakeClientFactory: @@ -338,10 +359,14 @@ class FakeApiPb2: def __init__(self) -> None: self.SecretDeleteRequest = self._secret_delete_request + self.ImageDeleteRequest = self._image_delete_request def _secret_delete_request(self, *, secret_id: str) -> Call: return Call("SecretDeleteRequest", (), {"secret_id": secret_id}) + def _image_delete_request(self, *, image_id: str) -> Call: + return Call("ImageDeleteRequest", (), {"image_id": image_id}) + class FakeSynchronizer: def wrap(self, fn: Any) -> Any: @@ -854,3 +879,124 @@ def test_an_authentication_failure_is_a_provider_error_not_a_raw_exception(self) with pytest.raises(EnvironmentsError) as raised: a_builder(modal=modal).exists(an_artifact(provider_artifact_id="im-abc123")) assert raised.value.code.code == PROVIDER_ERROR.code + + +class _Layer: + """One image in a chain, as `_intermediates_of` reads it.""" + + def __init__(self, *, object_id: Any = None, deps: Any = None) -> None: + self.object_id = object_id + self.deps = deps if deps is not None else (lambda: ()) + + +def _builder_with_client( + *, image_delete_errors: dict[str, Exception] | None = None +) -> tuple[Builder, FakeClient]: + """A builder whose client this test can read the delete calls back from.""" + client = FakeClient(image_delete_errors=image_delete_errors) + modal = FakeModalModule() + modal.Client = type( + "_C", + (), + { + "from_credentials": staticmethod(lambda *_: client), + "from_env": staticmethod(lambda: client), + }, + )() + return a_builder(modal=modal), client + + +# -- the layers a build leaves behind ----------------------------------------------------------- + + +class TestTheIntermediateLayers: + """What `build` records and `delete` collects (E2-05, E2-09). + + Each chained builder call leaves an image with an id of its own, and + deleting the artifact does not delete them. Modal offers no call that + lists an account's images — `ImageGetOrCreate`, `ImageFromId`, + `ImageGetByTag`, `ImageListTags`, `ImageTagRevisions`, `ImagePublish` and + `ImageDelete`, and nothing that enumerates — so an intermediate nobody + wrote down at build time can never be found again. + """ + + def test_every_layer_under_the_artifact_is_recorded_deepest_first(self) -> None: + from code_sandboxes.environments.adapters.modal import _intermediates_of + + base = _Layer(object_id="im-base", deps=lambda: ()) + middle = _Layer(object_id="im-middle", deps=lambda: (base,)) + built = _Layer(object_id="im-built", deps=lambda: (middle,)) + # The built image is the artifact, not an intermediate. + assert _intermediates_of(built) == ("im-base", "im-middle") + + def test_a_layer_with_no_id_is_not_recorded(self) -> None: + """Only a hydrated layer has an id worth writing down.""" + from code_sandboxes.environments.adapters.modal import _intermediates_of + + unbuilt = _Layer(object_id=None, deps=lambda: ()) + built = _Layer(object_id="im-built", deps=lambda: (unbuilt,)) + assert _intermediates_of(built) == () + + def test_a_chain_that_cannot_be_walked_is_no_layers_not_a_failure(self) -> None: + """A layer list is never worth failing a build over.""" + from code_sandboxes.environments.adapters.modal import _intermediates_of + + def _explode() -> Any: + raise RuntimeError("the SDK changed shape") + + assert _intermediates_of(_Layer(object_id="im-1", deps=_explode)) == () + + def test_delete_removes_the_intermediates_then_the_artifact(self) -> None: + """The artifact last: an intermediate is only reachable while the + record naming it survives.""" + builder, client = _builder_with_client() + builder.delete( + an_artifact( + intermediates=("im-base", "im-middle"), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-middle", "im-built"] + + def test_a_layer_already_gone_is_success(self) -> None: + """A replay of a collection deletes the same set again with no harm.""" + builder, client = _builder_with_client( + image_delete_errors={"im-base": FakeNotFoundError("gone")} + ) + builder.delete( + an_artifact( + intermediates=("im-base", "im-middle"), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-middle", "im-built"] + + def test_a_layer_that_is_not_ours_is_stepped_over(self) -> None: + """Modal's own `debian_slim` answers PermissionDenied, live on + 2026-09-17: an image somebody else owns was never ours to collect.""" + builder, client = _builder_with_client( + image_delete_errors={ + "im-base": RuntimeError("You don't have permission to modify Image 'im-base'") + } + ) + builder.delete( + an_artifact( + intermediates=("im-base",), + provider_artifact_id="im-built", + immutable_reference="im-built", + ) + ) + assert client.stub.image_delete_calls == ["im-base", "im-built"] + + def test_the_artifacts_own_refusal_is_raised(self) -> None: + """A layer's refusal is survivable; the artifact's is the whole point.""" + builder, _ = _builder_with_client( + image_delete_errors={"im-built": RuntimeError("modal is away")} + ) + with pytest.raises(EnvironmentsError) as raised: + builder.delete( + an_artifact(provider_artifact_id="im-built", immutable_reference="im-built") + ) + assert raised.value.code is PROVIDER_ERROR From 62790ae41c24abc63b2a98f987d7f47cd9bff23d Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 08:22:22 +0200 Subject: [PATCH 14/22] secrets --- code_sandboxes/environments/build_secrets.py | 172 ++++++++++++++++++- 1 file changed, 171 insertions(+), 1 deletion(-) diff --git a/code_sandboxes/environments/build_secrets.py b/code_sandboxes/environments/build_secrets.py index de0abd3..4bea149 100644 --- a/code_sandboxes/environments/build_secrets.py +++ b/code_sandboxes/environments/build_secrets.py @@ -25,10 +25,17 @@ from __future__ import annotations +import base64 +import json import os from typing import Any -from .errors import BUILD_SECRET_UNAVAILABLE, EnvironmentsError +from .errors import ( + BUILD_SECRET_UNAVAILABLE, + CAPABILITY_UNSUPPORTED, + PROVIDER_ERROR, + EnvironmentsError, +) from .spec import BuildSecret __all__ = [ @@ -150,3 +157,166 @@ def resolve_build_secret( retryable=False, ) return value + + +#: The managed variants an owner keeps a credential for (D-8). `datalayer` is +#: not one: the platform's own builder uses the platform's own registry. +PROVIDER_VARIANTS = frozenset({"daytona", "e2b", "modal"}) + + +def resolve_provider_credential( + variant: str, + *, + owner_uid: str, + iam_url: str | None = None, + api_key: str | None = None, + timeout: float = 10.0, + transport: Any = None, +) -> dict[str, str]: + """The owner's own credential for a managed variant (PLAN_ENVS.md E2-01, D-8). + + A managed artifact exists only in the account the credential opens, so the + build runs with the owner's keys and records a non-secret fingerprint of + that account. The value IAM holds is a JSON object of the environment + names the provider's own SDK reads — ``DAYTONA_API_KEY``, + ``E2B_API_KEY``, ``MODAL_TOKEN_ID`` and ``MODAL_TOKEN_SECRET``, and + optionally the account names ``environments/accounts.py`` reads to + fingerprint it — because that is the shape ``BuildCredential`` carries and + every adapter already reads. + + Raises ``DL_ENV_CAPABILITY_UNSUPPORTED`` — not retryable — when the owner + has no credential for this variant. A build must be told it cannot run in + an account nobody configured, rather than falling back to whatever keys + the worker happens to hold: that would put one owner's artifact in + another's account, which is the one thing D-8 exists to prevent. + """ + variant = (variant or "").strip().lower() + if variant not in PROVIDER_VARIANTS: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"`{variant}` is not a managed variant a credential is kept for", + detail={"variant": variant}, + retryable=False, + ) + key = api_key if api_key is not None else os.environ.get(IAM_API_KEY_VARIABLE, "") + if not key: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"No {IAM_API_KEY_VARIABLE} to ask IAM for the {variant} credential with", + detail={"variant": variant, "missing": IAM_API_KEY_VARIABLE}, + retryable=False, + ) + base = (iam_url if iam_url is not None else os.environ.get(IAM_URL_VARIABLE, "")).rstrip("/") + if not base: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"No {IAM_URL_VARIABLE} to ask for the {variant} credential", + detail={"variant": variant, "missing": IAM_URL_VARIABLE}, + retryable=False, + ) + try: + import httpx + except ImportError as error: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No HTTP client to resolve a provider credential with: install " + "`code-sandboxes[environments-builder]`", + detail={"variant": variant, "missing": "httpx"}, + retryable=False, + ) from error + url = f"{base}/api/iam/v1/secrets/provider/{variant}/value" + try: + with httpx.Client(transport=transport, timeout=timeout) as client: + response = client.get( + url, + params={"owner_uid": owner_uid}, + headers={"X-API-Key": key}, + ) + except httpx.HTTPError as error: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM could not be reached for the {variant} credential: {error}", + detail={"variant": variant}, + retryable=True, + ) from error + if response.status_code == 404: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"This owner has no {variant} credential: a managed build runs in " + f"their own account, so one must be kept before {variant} can build", + detail={"variant": variant, "status": 404}, + retryable=False, + ) + if response.status_code != 200: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM refused the {variant} credential: {response.status_code}", + detail={"variant": variant, "status": response.status_code}, + retryable=True, + ) + try: + body = response.json() + except ValueError as error: + raise EnvironmentsError( + PROVIDER_ERROR, + f"IAM answered the {variant} credential with a body that is not valid JSON", + detail={"variant": variant}, + retryable=True, + ) from error + return _provider_secrets_of(body.get("value") if isinstance(body, dict) else None, variant) + + +def _provider_secrets_of(value: Any, variant: str) -> dict[str, str]: + """The environment names a provider credential carries, from what IAM held. + + Accepts the value as JSON, and as base64 of that JSON, because the secret + routes say values arrive base64-encoded from their clients while nothing + enforces it — a credential that cannot be read is a build that cannot run, + so both shapes are read rather than one being assumed. + """ + if not isinstance(value, str) or not value.strip(): + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"IAM answered the {variant} credential with no usable value", + detail={"variant": variant}, + retryable=False, + ) + text = value.strip() + parsed: Any = None + for candidate in (text, _decoded(text)): + if not candidate: + continue + try: + parsed = json.loads(candidate) + except ValueError: + continue + break + if not isinstance(parsed, dict) or not parsed: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"The {variant} credential is not a JSON object of environment names; " + "a provider credential holds the names that provider's own SDK reads", + detail={"variant": variant}, + retryable=False, + ) + secrets = { + str(name): str(item) + for name, item in parsed.items() + if str(name).strip() and str(item).strip() + } + if not secrets: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + f"The {variant} credential names nothing", + detail={"variant": variant}, + retryable=False, + ) + return secrets + + +def _decoded(text: str) -> str: + """`text` as base64, or empty when it is not.""" + try: + return base64.b64decode(text, validate=True).decode("utf-8") + except Exception: # noqa: BLE001 - not base64 is an answer, not a failure + return "" From 7e4eb1b495c6b08949dd621175f006d68a725d96 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 08:28:55 +0200 Subject: [PATCH 15/22] environments: resolve the owner's own provider credential (E2-01, D-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A managed artifact exists only in the account its credential opens, so a managed build needs the owner's own E2B, Daytona or Modal keys — and nothing could fetch them, which is why every managed build refused. `resolve_provider_credential` asks IAM by variant and owner, the same shape `resolve_build_secret` already uses, and answers the environment names the provider's own SDK reads, which is what `BuildCredential` carries and every adapter already reads. Both a JSON value and base64 of that JSON are accepted: the secret routes say clients encode values and nothing enforces it, and a credential that cannot be read is a build that cannot run. An owner with no credential is refused by name and not retried. Never a fallback to whatever keys the worker holds: that would put one owner's artifact in whatever account the deployment happens to have, which is the one thing D-8 exists to prevent. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/environments/build_secrets.py | 2 +- tests/test_environment_build_secrets.py | 124 ++++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/code_sandboxes/environments/build_secrets.py b/code_sandboxes/environments/build_secrets.py index 4bea149..7d70772 100644 --- a/code_sandboxes/environments/build_secrets.py +++ b/code_sandboxes/environments/build_secrets.py @@ -318,5 +318,5 @@ def _decoded(text: str) -> str: """`text` as base64, or empty when it is not.""" try: return base64.b64decode(text, validate=True).decode("utf-8") - except Exception: # noqa: BLE001 - not base64 is an answer, not a failure + except Exception: return "" diff --git a/tests/test_environment_build_secrets.py b/tests/test_environment_build_secrets.py index e28cf0d..15074f2 100644 --- a/tests/test_environment_build_secrets.py +++ b/tests/test_environment_build_secrets.py @@ -11,6 +11,9 @@ from __future__ import annotations +import base64 +import json + import httpx import pytest @@ -18,8 +21,13 @@ IAM_API_KEY_VARIABLE, IAM_URL_VARIABLE, resolve_build_secret, + resolve_provider_credential, +) +from code_sandboxes.environments.errors import ( + BUILD_SECRET_UNAVAILABLE, + CAPABILITY_UNSUPPORTED, + EnvironmentsError, ) -from code_sandboxes.environments.errors import BUILD_SECRET_UNAVAILABLE, EnvironmentsError from code_sandboxes.environments.spec import BuildSecret SECRET = BuildSecret(id="dlsec_01J9BUILDSECRET0000000000", name="PIP_TOKEN") @@ -215,3 +223,117 @@ def handler(request: httpx.Request) -> httpx.Response: value = resolve_build_secret(SECRET, owner_uid=OWNER, transport=httpx.MockTransport(handler)) assert value == "tok-1" + + +# -- the owner's own provider credential (E2-01, D-8) ------------------------------------------- + + +def _credential_transport(handler) -> httpx.MockTransport: + return httpx.MockTransport(handler) + + +def test_the_credential_is_asked_for_by_variant_and_owner() -> None: + """A build knows whose it is and which variant it is building, never a secret id.""" + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response( + 200, + json={"provider": "daytona", "value": json.dumps({"DAYTONA_API_KEY": "k"})}, + ) + + secrets = resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="worker-key", + transport=_credential_transport(handler), + ) + assert secrets == {"DAYTONA_API_KEY": "k"} + (request,) = seen + assert request.url.path == "/api/iam/v1/secrets/provider/daytona/value" + assert request.url.params["owner_uid"] == "owner-1" + # The worker's own key, never a person's token. + assert request.headers["X-API-Key"] == "worker-key" + assert "authorization" not in {name.lower() for name in request.headers} + + +def test_a_base64_value_is_read_too() -> None: + """The secret routes say clients encode values; nothing enforces it.""" + raw = json.dumps({"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"}) + + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"value": base64.b64encode(raw.encode()).decode()}, + ) + + assert resolve_provider_credential( + "e2b", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) == {"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"} + + +def test_an_owner_with_no_credential_is_refused_by_name_and_not_retried() -> None: + """Never a fallback to whatever keys the worker holds: that is the one + thing D-8 exists to prevent.""" + + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(404, json={"detail": "No credential for that provider"}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "modal", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED + assert raised.value.retryable is False + assert "their own account" in str(raised.value) + + +def test_iam_being_away_is_retryable() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(503, json={"detail": "away"}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.retryable is True + + +def test_datalayer_is_not_a_variant_a_credential_is_kept_for() -> None: + """The platform's own builder uses the platform's own registry.""" + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "datalayer", owner_uid="owner-1", iam_url="https://iam.example", api_key="k" + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED + + +def test_a_credential_that_names_nothing_is_refused() -> None: + for value in ("{}", "not json", '"a string"', ""): + + def handler(_request: httpx.Request, value: str = value) -> httpx.Response: + return httpx.Response(200, json={"value": value}) + + with pytest.raises(EnvironmentsError) as raised: + resolve_provider_credential( + "daytona", + owner_uid="owner-1", + iam_url="https://iam.example", + api_key="k", + transport=_credential_transport(handler), + ) + assert raised.value.code is CAPABILITY_UNSUPPORTED, value From 2c63a9ed884c2849b4c1b21838a6a23193ef5d68 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 08:52:12 +0200 Subject: [PATCH 16/22] environments: read the provider keys the owner already keeps (E2-01, D-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version of this invented a convention — one secret per provider holding a JSON blob — beside one that already works. The owner keeps DAYTONA_API_KEY, E2B_API_KEY, MODAL_TOKEN_ID and MODAL_TOKEN_SECRET as ordinary secrets named after the environment variables their own SDKs read, which is exactly the shape a build credential carries. So IAM gathers the credential from those and answers a mapping, decoded, and the resolver takes it as given: the worker is not asked to know how a secret is stored. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/environments/build_secrets.py | 45 ++++---------------- tests/test_environment_build_secrets.py | 20 +++------ 2 files changed, 16 insertions(+), 49 deletions(-) diff --git a/code_sandboxes/environments/build_secrets.py b/code_sandboxes/environments/build_secrets.py index 7d70772..2a3e9f3 100644 --- a/code_sandboxes/environments/build_secrets.py +++ b/code_sandboxes/environments/build_secrets.py @@ -25,8 +25,6 @@ from __future__ import annotations -import base64 -import json import os from typing import Any @@ -267,41 +265,24 @@ def resolve_provider_credential( def _provider_secrets_of(value: Any, variant: str) -> dict[str, str]: - """The environment names a provider credential carries, from what IAM held. + """The environment names a provider credential carries, as IAM answered. - Accepts the value as JSON, and as base64 of that JSON, because the secret - routes say values arrive base64-encoded from their clients while nothing - enforces it — a credential that cannot be read is a build that cannot run, - so both shapes are read rather than one being assumed. + A mapping of names to values: IAM gathers them from the secrets the owner + already keeps, one per environment variable, and decodes them, so the + worker is not asked to know how a secret is stored. """ - if not isinstance(value, str) or not value.strip(): + if not isinstance(value, dict) or not value: raise EnvironmentsError( CAPABILITY_UNSUPPORTED, - f"IAM answered the {variant} credential with no usable value", - detail={"variant": variant}, - retryable=False, - ) - text = value.strip() - parsed: Any = None - for candidate in (text, _decoded(text)): - if not candidate: - continue - try: - parsed = json.loads(candidate) - except ValueError: - continue - break - if not isinstance(parsed, dict) or not parsed: - raise EnvironmentsError( - CAPABILITY_UNSUPPORTED, - f"The {variant} credential is not a JSON object of environment names; " - "a provider credential holds the names that provider's own SDK reads", + f"IAM answered the {variant} credential with no usable value; a " + "provider credential is the environment names that provider's own " + "SDK reads", detail={"variant": variant}, retryable=False, ) secrets = { str(name): str(item) - for name, item in parsed.items() + for name, item in value.items() if str(name).strip() and str(item).strip() } if not secrets: @@ -312,11 +293,3 @@ def _provider_secrets_of(value: Any, variant: str) -> dict[str, str]: retryable=False, ) return secrets - - -def _decoded(text: str) -> str: - """`text` as base64, or empty when it is not.""" - try: - return base64.b64decode(text, validate=True).decode("utf-8") - except Exception: - return "" diff --git a/tests/test_environment_build_secrets.py b/tests/test_environment_build_secrets.py index 15074f2..85393c0 100644 --- a/tests/test_environment_build_secrets.py +++ b/tests/test_environment_build_secrets.py @@ -11,9 +11,6 @@ from __future__ import annotations -import base64 -import json - import httpx import pytest @@ -240,7 +237,7 @@ def handler(request: httpx.Request) -> httpx.Response: seen.append(request) return httpx.Response( 200, - json={"provider": "daytona", "value": json.dumps({"DAYTONA_API_KEY": "k"})}, + json={"provider": "daytona", "value": {"DAYTONA_API_KEY": "k"}}, ) secrets = resolve_provider_credential( @@ -259,15 +256,12 @@ def handler(request: httpx.Request) -> httpx.Response: assert "authorization" not in {name.lower() for name in request.headers} -def test_a_base64_value_is_read_too() -> None: - """The secret routes say clients encode values; nothing enforces it.""" - raw = json.dumps({"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"}) +def test_the_account_names_come_through_beside_the_key() -> None: + """`environments/accounts.py` fingerprints the account from these, so a + credential is more than the key that opens it.""" def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - json={"value": base64.b64encode(raw.encode()).decode()}, - ) + return httpx.Response(200, json={"value": {"E2B_API_KEY": "k", "E2B_TEAM_ID": "team"}}) assert resolve_provider_credential( "e2b", @@ -323,9 +317,9 @@ def test_datalayer_is_not_a_variant_a_credential_is_kept_for() -> None: def test_a_credential_that_names_nothing_is_refused() -> None: - for value in ("{}", "not json", '"a string"', ""): + for value in ({}, "a string", None, {"": "v"}): - def handler(_request: httpx.Request, value: str = value) -> httpx.Response: + def handler(_request: httpx.Request, value=value) -> httpx.Response: return httpx.Response(200, json={"value": value}) with pytest.raises(EnvironmentsError) as raised: From 9435e959fe1c3edf3c49e911a3fee17a714e1460 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 08:54:59 +0200 Subject: [PATCH 17/22] release 1.9.19: the owner's own provider credential, and Modal's layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E2-01's remaining half, which is what kept every managed build from running at all: `resolve_provider_credential` reads the keys an owner already keeps — DAYTONA_API_KEY, E2B_API_KEY, MODAL_TOKEN_ID, MODAL_TOKEN_SECRET — so a build runs in their own account and never in whatever account the worker happens to hold. Modal records the intermediate layers a build leaves and `delete` collects them, because Modal has no call that lists an account's images: an intermediate nobody writes down can never be found again. Licences are read from an SBOM (SPDX and CycloneDX) for a publication to carry; the registry's scanner reports vulnerabilities, not licences. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index ed7b561..9e8e7a4 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.18" +__version__ = "1.9.19" From fae5594e63a84dc2e492a658c886fe537a27705d Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 09:39:20 +0200 Subject: [PATCH 18/22] environments: a Dockerfile in the spec, refused per variant (E3-03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `build.source: dockerfile` had nowhere to put the file, so nothing could read it and no variant could say whether it would honour it. `DockerfileSpec` carries the text inline, the way `DependencyFileSpec` carries a requirements.txt: a file the author brings lives in the spec, so the contract and every capability report read it before anything is queued. The bounded upload E3-03 also describes is for the build *context* — the extra files a COPY needs — which `validate_build_context` already checks. All three managed variants take a Dockerfile, each through its own door, and each declares what its own builder will not honour: E2B VOLUME EXPOSE HEALTHCHECK SHELL ONBUILD STOPSIGNAL LABEL MAINTAINER Modal ONBUILD STOPSIGNAL VOLUME Daytona none beyond the contract's own E2B's list is read from its SDK rather than guessed: `e2b.template.dockerfile_parser` branches on FROM, RUN, COPY, ADD, WORKDIR, USER, ENV, ARG, CMD and ENTRYPOINT, and for anything else **prints `Unsupported instruction` and carries on** — so a template built from a Dockerfile naming one comes back without it and reports success. That is the case a capability report exists for. Daytona hands the text to a real Docker builder, so Docker's grammar is its limit. Every refusal names its line, at `validate`, because a Dockerfile is somebody's file and "it was refused" is not a reason. Co-Authored-By: Claude Opus 5 (1M context) --- .../environments/adapters/daytona.py | 7 +- code_sandboxes/environments/adapters/e2b.py | 19 ++++- .../environments/adapters/managed.py | 38 +++++++++- code_sandboxes/environments/adapters/modal.py | 2 +- code_sandboxes/environments/spec.py | 62 +++++++++++++---- schemas/environment-v1alpha1.json | 24 +++++++ tests/test_environment_managed_builders.py | 69 +++++++++++++++---- tests/test_environment_spec.py | 38 +++++++++- 8 files changed, 224 insertions(+), 35 deletions(-) diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index f43dc1a..e2d702c 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -179,7 +179,12 @@ class Builder(ManagedBuilder): title = "Daytona" #: A `packages` list and, for conda (E3-02), an `environment.yml` #: dependency file installed with `micromamba`. - build_sources = ("packages", "dependencyFile") + build_sources = ("packages", "dependencyFile", "dockerfile") + #: None beyond the contract's own (E3-03): `Image.from_dockerfile` keeps + #: the Dockerfile text as it is and Daytona builds it on a real Docker + #: builder, so the grammar it accepts is Docker's. Checked in the SDK on + #: 2026-09-17. + forbidden_instructions = () dependency_formats = ("conda",) #: Daytona runs GPUs, on its own hardware and the owner's account (E2-17). #: This builder does not build one yet: see `_own_findings`. diff --git a/code_sandboxes/environments/adapters/e2b.py b/code_sandboxes/environments/adapters/e2b.py index 875eca2..0b168d6 100644 --- a/code_sandboxes/environments/adapters/e2b.py +++ b/code_sandboxes/environments/adapters/e2b.py @@ -199,7 +199,24 @@ class Builder(ManagedBuilder): title = "E2B" #: A `packages` list and, for conda (E3-02), an `environment.yml` #: dependency file installed with `micromamba`. - build_sources = ("packages", "dependencyFile") + build_sources = ("packages", "dependencyFile", "dockerfile") + #: What E2B's own Dockerfile parser does not handle (E3-03). Read from the + #: SDK on 2026-09-17: `e2b.template.dockerfile_parser` branches on FROM, + #: RUN, COPY, ADD, WORKDIR, USER, ENV, ARG, CMD and ENTRYPOINT, and for + #: anything else **prints `Unsupported instruction` and carries on**. So a + #: template built from a Dockerfile naming one of these comes back without + #: it and reports success — which is exactly what a capability report + #: exists to prevent. + forbidden_instructions = ( + "VOLUME", + "EXPOSE", + "HEALTHCHECK", + "SHELL", + "ONBUILD", + "STOPSIGNAL", + "LABEL", + "MAINTAINER", + ) dependency_formats = ("conda",) #: Firecracker microVMs: no GPU passthrough. gpu = False diff --git a/code_sandboxes/environments/adapters/managed.py b/code_sandboxes/environments/adapters/managed.py index 668c4fc..8425037 100644 --- a/code_sandboxes/environments/adapters/managed.py +++ b/code_sandboxes/environments/adapters/managed.py @@ -73,7 +73,11 @@ class ManagedBuilder: #: built for a managed variant yet (E3-01), so it is not listed here. dependency_formats: tuple[str, ...] = () package_managers: tuple[str, ...] = ("uv", "pip") - #: Dockerfile instructions its own builder does not implement (§6). + #: Dockerfile instructions its own builder does not implement (§6, E3-03). + #: Checked against a `dockerfile`-sourced spec at `validate`, before + #: anything is queued — the point being to refuse an instruction a variant + #: would otherwise drop, rather than hand back an image quietly missing + #: whatever it asked for. forbidden_instructions: tuple[str, ...] = () #: Where its artifacts live; empty when the variant is regionless. regions: tuple[str, ...] = () @@ -128,6 +132,8 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) elif spec.build.source == "dependencyFile": findings.extend(self._dependency_file_findings(spec)) + elif spec.build.source == "dockerfile": + findings.extend(self._dockerfile_findings(spec)) if spec.packages.python.manager not in self.package_managers: findings.append( CapabilityFinding( @@ -269,6 +275,36 @@ def delete(self, artifact: ArtifactReference) -> None: # -- helpers for the subclasses ------------------------------------------- + def _dockerfile_findings(self, spec: Any) -> list[CapabilityFinding]: + """Every instruction in the spec's Dockerfile this variant would not honour. + + The contract's own refusals are `spec.py`'s to make and apply to every + variant alike; this is the narrower question of what *this* builder + does with a Dockerfile it accepts. Each is named with its line, since + a Dockerfile is somebody's file. + """ + if not self.forbidden_instructions: + return [] + dockerfile = getattr(spec.build, "dockerfile", None) + content = getattr(dockerfile, "content", "") or "" + if not content.strip(): + return [] + from ..contract import parse_dockerfile + + refused = set(self.forbidden_instructions) + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"line {instruction.line}: {self.title} does not implement " + f"`{instruction.keyword}`" + ), + field="spec.build.dockerfile.content", + ) + for instruction in parse_dockerfile(content) + if instruction.keyword in refused + ] + @staticmethod def _spec_finding(message: str, field: str) -> CapabilityFinding: return CapabilityFinding(code=SPEC_INVALID.code, message=message, field=field) diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index 3c557ab..4fa703a 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -286,7 +286,7 @@ class Builder(ManagedBuilder): title = "Modal" #: A `packages` list and, for conda (E3-02), an `environment.yml` #: dependency file installed with `micromamba_install`. - build_sources = ("packages", "dependencyFile") + build_sources = ("packages", "dependencyFile", "dockerfile") dependency_formats = ("conda",) #: Modal runs GPUs, in the owner's workspace (E2-17). This builder does #: not build one yet: see `build`'s own guard. diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index d180f9a..5a3e1b2 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -55,10 +55,10 @@ "BUILD_SOURCES", "GPU_SIZE_CLASSES", "KIND", + "PUBLIC_PACKAGE_INDEX_HOSTS", "SIZE_CLASSES", "SUPPORTED_BUILD_SOURCES", "VARIANTS", - "PUBLIC_PACKAGE_INDEX_HOSTS", "Accelerator", "ArtifactStatus", "Base", @@ -174,14 +174,12 @@ class Platform(_Model): #: reached with a credential the public does not hold. Matched on host, so the #: trailing `/simple` or its absence never decides it. `pypi.org` is the index; #: `files.pythonhosted.org` is where its wheels are served from. -PUBLIC_PACKAGE_INDEX_HOSTS = frozenset( - {"pypi.org", "files.pythonhosted.org"} -) +PUBLIC_PACKAGE_INDEX_HOSTS = frozenset({"pypi.org", "files.pythonhosted.org"}) def _package_index_host(url: str) -> str: """The host an index URL names, lower-cased and without its port, or `""`.""" - from urllib.parse import urlsplit # noqa: PLC0415 + from urllib.parse import urlsplit try: return (urlsplit(url).hostname or "").lower() @@ -214,9 +212,7 @@ def index_is_public(url: str) -> bool: "msys2", } ) -PUBLIC_CONDA_CHANNEL_HOSTS = frozenset( - {"conda.anaconda.org", "repo.anaconda.com", "anaconda.org"} -) +PUBLIC_CONDA_CHANNEL_HOSTS = frozenset({"conda.anaconda.org", "repo.anaconda.com", "anaconda.org"}) def channel_is_public(channel: str) -> bool: @@ -344,6 +340,25 @@ class DependencyFileSpec(_Model): lock_content: str = "" +class DockerfileSpec(_Model): + """The Dockerfile a `dockerfile`-sourced version builds from (E3-03). + + Inline, the way ``DependencyFileSpec`` carries a ``requirements.txt``: a + file the author brings lives in the spec, so ``check_dockerfile`` and + every variant's capability report can read it **before** anything is + queued — which is the whole point of refusing an instruction a variant + does not implement at ``validate`` rather than halfway through a build. + + The build *context* — the extra files a ``COPY`` needs — is the separate + upload E3-03 describes, bounded and checked by ``validate_build_context``. + A Dockerfile with no context is the common case and needs no upload at + all. + """ + + #: The Dockerfile text. + content: str = "" + + class ImageSourceSpec(_Model): """An existing OCI image, imported as the build's base (E3-04). @@ -370,6 +385,7 @@ class ImageSourceSpec(_Model): class BuildSpec(_Model): source: Literal["packages", "dependencyFile", "dockerfile", "image"] = "packages" dependency_file: DependencyFileSpec | None = None + dockerfile: DockerfileSpec | None = None image: ImageSourceSpec | None = None @@ -486,6 +502,28 @@ def parse_requirements_txt(text: str) -> list[str]: return lines +def _dockerfile_findings(dockerfile: DockerfileSpec | None) -> list[SpecFinding]: + """What a `dockerfile`-sourced version must carry, and what the contract refuses. + + The contract's own refusals are reported here, at `validate`, rather than + at build time: a `FROM` that is not an approved base, a privileged build, + the host network, a Docker socket mount, a host bind mount, and every + instruction §6 does not allow. Each is named with its line, because a + Dockerfile is somebody's file and "it was refused" is not a reason. + """ + field = "spec.build.dockerfile" + if dockerfile is None: + return [SpecFinding(field, "is required when `spec.build.source` is `dockerfile`")] + if not dockerfile.content.strip(): + return [SpecFinding(f"{field}.content", "is empty; it is the Dockerfile text")] + from .contract import validate_dockerfile + + return [ + SpecFinding(f"{field}.content", f"line {finding.line}: {finding.message}") + for finding in validate_dockerfile(dockerfile.content) + ] + + def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> list[SpecFinding]: field = "spec.build.dependencyFile" if dependency_file is None: @@ -644,6 +682,8 @@ def spec_findings( ) if spec.build.source == "dependencyFile": findings.extend(_dependency_file_findings(spec.build.dependency_file)) + elif spec.build.source == "dockerfile": + findings.extend(_dockerfile_findings(spec.build.dockerfile)) elif spec.build.source == "image": findings.extend(_image_findings(spec.build.image)) @@ -951,11 +991,7 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: PUBLICATION_BLOCKED, ) ) - private = [ - url - for url in environment.spec.packages.python.indexes - if not index_is_public(url) - ] + private = [url for url in environment.spec.packages.python.indexes if not index_is_public(url)] if private: findings.append( SpecFinding( diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index 2c44291..4b7040a 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -95,6 +95,17 @@ ], "default": null }, + "dockerfile": { + "anyOf": [ + { + "$ref": "#/$defs/DockerfileSpec" + }, + { + "type": "null" + } + ], + "default": null + }, "image": { "anyOf": [ { @@ -219,6 +230,19 @@ "title": "DependencyFileSpec", "type": "object" }, + "DockerfileSpec": { + "additionalProperties": false, + "description": "The Dockerfile a `dockerfile`-sourced version builds from (E3-03).\n\nInline, the way ``DependencyFileSpec`` carries a ``requirements.txt``: a\nfile the author brings lives in the spec, so ``check_dockerfile`` and\nevery variant's capability report can read it **before** anything is\nqueued \u2014 which is the whole point of refusing an instruction a variant\ndoes not implement at ``validate`` rather than halfway through a build.\n\nThe build *context* \u2014 the extra files a ``COPY`` needs \u2014 is the separate\nupload E3-03 describes, bounded and checked by ``validate_build_context``.\nA Dockerfile with no context is the common case and needs no upload at\nall.", + "properties": { + "content": { + "default": "", + "title": "Content", + "type": "string" + } + }, + "title": "DockerfileSpec", + "type": "object" + }, "EnvironmentSpec": { "additionalProperties": false, "properties": { diff --git a/tests/test_environment_managed_builders.py b/tests/test_environment_managed_builders.py index 7816645..8c479d6 100644 --- a/tests/test_environment_managed_builders.py +++ b/tests/test_environment_managed_builders.py @@ -55,11 +55,7 @@ def environment(**spec: Any) -> Environment: #: A `dependencyFile` conda source: an `environment.yml` a managed variant #: builds with `micromamba` (E3-02). CONDA_ENVIRONMENT_YML = ( - "name: geo\n" - "channels: [conda-forge]\n" - "dependencies:\n" - " - python=3.13\n" - " - gdal\n" + "name: geo\nchannels: [conda-forge]\ndependencies:\n - python=3.13\n - gdal\n" ) @@ -102,15 +98,33 @@ def test_e2b_has_no_gpu_and_the_other_two_do(self) -> None: assert get_builder("modal").capabilities().supports_gpu is True assert get_builder("daytona").capabilities().supports_gpu is True - def test_only_modal_forbids_instructions_its_builder_never_implemented(self) -> None: - """Modal implements its own Dockerfile builder; the others hand a - Dockerfile to BuildKit, which implements all of them.""" + def test_each_variant_forbids_what_its_own_builder_will_not_honour(self) -> None: + """Modal implements its own Dockerfile builder, and E2B parses a + Dockerfile into Template SDK calls; Daytona hands the text to a real + Docker builder, which implements all of them. + + E2B's list is read from its SDK (E3-03, 2026-09-17): + `e2b.template.dockerfile_parser` branches on FROM, RUN, COPY, ADD, + WORKDIR, USER, ENV, ARG, CMD and ENTRYPOINT, and for anything else + **prints `Unsupported instruction` and carries on** — so a template + built from a Dockerfile naming one of these comes back without it and + reports success. That is the case a capability report exists for. + """ assert set(get_builder("modal").capabilities().forbidden_instructions) == { "ONBUILD", "STOPSIGNAL", "VOLUME", } - assert get_builder("e2b").capabilities().forbidden_instructions == () + assert set(get_builder("e2b").capabilities().forbidden_instructions) == { + "VOLUME", + "EXPOSE", + "HEALTHCHECK", + "SHELL", + "ONBUILD", + "STOPSIGNAL", + "LABEL", + "MAINTAINER", + } assert get_builder("daytona").capabilities().forbidden_instructions == () def test_each_one_bounds_how_long_a_build_may_take(self) -> None: @@ -120,10 +134,13 @@ def test_each_one_bounds_how_long_a_build_may_take(self) -> None: seconds = get_builder(variant).capabilities().max_build_seconds assert seconds and 0 < seconds <= 60 * 60, variant - def test_this_phase_builds_a_package_list_and_a_conda_file(self) -> None: + def test_this_phase_builds_a_package_list_a_conda_file_and_a_dockerfile(self) -> None: + """All three take a Dockerfile (E3-03), each through its own door: + E2B parses one into Template SDK calls, Daytona hands the text to a + real Docker builder, Modal builds it with its own frontend.""" for variant in MANAGED: sources = get_builder(variant).capabilities().build_sources - assert sources == ("packages", "dependencyFile"), variant + assert sources == ("packages", "dependencyFile", "dockerfile"), variant # -- what each one refuses ------------------------------------------------------ @@ -230,10 +247,29 @@ def test_other_variables_are_left_alone(self) -> None: def test_a_source_this_phase_does_not_build_is_refused_by_name(self) -> None: for variant in MANAGED: - report = get_builder(variant).validate(environment(build={"source": "dockerfile"})) + report = get_builder(variant).validate(environment(build={"source": "image"})) assert report.supported is False, variant - assert "`dockerfile` is not built for" in messages(report) - assert "it builds packages, dependencyFile" in messages(report) + assert "`image` is not built for" in messages(report) + assert "it builds packages, dependencyFile, dockerfile" in messages(report) + + def test_an_instruction_a_variant_would_drop_is_refused_with_its_line(self) -> None: + """The case this exists for: E2B's parser prints `Unsupported + instruction` and carries on, so without this the template comes back + missing what the Dockerfile asked for and the build reports success.""" + dockerfile = "FROM datalayer/python-cpu:2026.09\nRUN true\nVOLUME /data\n" + report = get_builder("e2b").validate( + environment(build={"source": "dockerfile", "dockerfile": {"content": dockerfile}}) + ) + assert report.supported is False + assert "line 3: E2B does not implement `VOLUME`" in messages(report) + + def test_a_dockerfile_a_variant_can_honour_is_accepted(self) -> None: + """Daytona builds the text as it is, so Docker's own grammar is the limit.""" + dockerfile = "FROM datalayer/python-cpu:2026.09\nVOLUME /data\nEXPOSE 8888\n" + report = get_builder("daytona").validate( + environment(build={"source": "dockerfile", "dockerfile": {"content": dockerfile}}) + ) + assert report.supported is True, messages(report) def test_a_conda_dependency_file_is_buildable_on_every_managed_variant(self) -> None: for variant in MANAGED: @@ -415,8 +451,11 @@ def test_every_variant_validates_with_no_provider_sdk_installed() -> None: assert builder.capabilities().variant == variant report = builder.validate(environment()) assert report.supported is True, f"{variant}: {messages(report)}" + # A refusal each variant still makes, and makes without an SDK. + # `dockerfile` stopped being one when E3-03 gave all three a door + # into it, so a managed variant is asked about `image` instead. refused = get_builder(variant).validate( - environment(build={"source": "dockerfile"}) + environment(build={"source": "image"}) if variant != "datalayer" else environment( resources={"sizeClass": "gpu-small", "accelerator": {"type": "A10G"}} diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index 88a8d9f..0740366 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -322,14 +322,46 @@ def test_an_invalid_field_outranks_something_unsupported() -> None: } -def test_a_dockerfile_source_is_accepted_and_keeps_its_own_base() -> None: - # E3-03: the base is the `FROM` its uploaded Dockerfile names, so - # `spec.base` is not checked against the approved table (as for `image`). +def a_dockerfile_document(content: str) -> dict[str, Any]: data = mutated("spec.build.source", "dockerfile") + data["spec"]["build"]["dockerfile"] = {"content": content} + return data + + +def test_a_dockerfile_source_is_accepted_and_keeps_its_own_base() -> None: + # E3-03: the base is the `FROM` the Dockerfile names, so `spec.base` is + # not checked against the approved table (as for `image`). + data = a_dockerfile_document("FROM datalayer/python-cpu:2026.09\nRUN true\n") data["spec"]["base"]["ref"] = "python" assert _codes(data) == {} +def test_a_dockerfile_source_without_a_dockerfile_is_refused() -> None: + """`source: dockerfile` with nowhere to read the file from is not a spec.""" + data = mutated("spec.build.source", "dockerfile") + assert _codes(data) == {"spec.build.dockerfile": "DL_ENV_SPEC_INVALID"} + + +def test_an_empty_dockerfile_is_refused() -> None: + assert _codes(a_dockerfile_document(" \n")) == { + "spec.build.dockerfile.content": "DL_ENV_SPEC_INVALID" + } + + +def test_the_contract_refuses_a_dockerfile_at_validate_naming_the_line() -> None: + """A Dockerfile is somebody's file: "it was refused" is not a reason. + + The refusals are the contract's own — an unapproved base, the host + network, a privileged build, a Docker socket mount — and they are made + here, before anything is queued, rather than partway through a build. + """ + data = a_dockerfile_document("FROM ubuntu:22.04\nRUN --network=host apt-get update\n") + assert _codes(data) == {"spec.build.dockerfile.content": "DL_ENV_SPEC_INVALID"} + messages = [finding.message for finding in spec_findings(parse_environment(data))] + assert any("line 1" in message and "approved" in message for message in messages) + assert any("line 2" in message and "host network" in message for message in messages) + + # -- Dependency files (E3-01) -------------------------------------------------- From 28fc022e4a09ce4bdf004bc11131183856a291e3 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 10:22:11 +0200 Subject: [PATCH 19/22] daytona: send a region Daytona knows, or none (E2-04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real Daytona build answered "Failed to create snapshot: Region not found". `region_id` was `request.region` — *this platform's* region, `r1` — which Daytona has never heard of. The region that scopes a snapshot is Daytona's, and the owner names it in `compatibility.regions`, which `validate` already refuses more than one of. With none named the field is left out and the account's own default decides, which is what every snapshot in a real Daytona account already has. Co-Authored-By: Claude Opus 5 (1M context) --- .../environments/adapters/daytona.py | 15 +++++++++++++- tests/test_environment_daytona_builder.py | 20 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index e2d702c..e17eb4b 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -424,6 +424,11 @@ def on_logs(line: str) -> None: self._log(line) resources = self._resources(sdk, request.size_class) + # A snapshot is region-scoped, and the region that scopes it is + # Daytona's, not this platform's. `validate` already refuses + # more than one, so the first is the only one. + declared = list(request.environment.spec.compatibility.regions) + region = declared[0] if declared else None try: snapshot = client.snapshot.create( sdk.CreateSnapshotParams( @@ -431,7 +436,15 @@ def on_logs(line: str) -> None: image=image, resources=resources, entrypoint=_CONTRACT_ENTRYPOINT, - region_id=request.region, + # Only a region Daytona knows. `request.region` is + # *Datalayer's* — `r1` — and sending it answered + # "Region not found" on the first real Daytona + # build (2026-09-17). The owner names a Daytona + # region in `compatibility.regions`; with none, + # the field is left out and the account's own + # default decides, which is what every snapshot + # in the owner's account already has. + **({"region_id": region} if region else {}), ), on_logs=on_logs, timeout=self.max_build_seconds, diff --git a/tests/test_environment_daytona_builder.py b/tests/test_environment_daytona_builder.py index 5ba666c..b2c9fa4 100644 --- a/tests/test_environment_daytona_builder.py +++ b/tests/test_environment_daytona_builder.py @@ -568,12 +568,28 @@ def test_resources_come_from_the_size_class( resources = daytona.client.snapshot.create_calls[0].args[0].resources assert (resources.cpu, resources.memory, resources.disk) == (cpu, memory, disk) - def test_the_region_is_passed_through(self) -> None: + def test_the_region_sent_is_daytonas_own_not_this_platforms(self) -> None: + """`request.region` is Datalayer's — `r1` — and Daytona answered + "Region not found" for it on the first real build (2026-09-17). + + The owner names a Daytona region in `compatibility.regions`; that is + the one that scopes the snapshot. + """ daytona = FakeDaytonaModule() - a_builder(daytona=daytona).build(a_request(region="eu")) + a_builder(daytona=daytona).build( + a_request(region="r1", spec={"compatibility": {"regions": ["eu"]}}) + ) params = daytona.client.snapshot.create_calls[0].args[0] assert params.region_id == "eu" + def test_with_no_region_named_the_account_default_decides(self) -> None: + """The field is left out rather than filled with something Daytona + does not know — which is what every snapshot in a real account has.""" + daytona = FakeDaytonaModule() + a_builder(daytona=daytona).build(a_request(region="r1")) + params = daytona.client.snapshot.create_calls[0].args[0] + assert params.region_id is None + def test_the_snapshot_is_named_after_the_environment_and_version(self) -> None: daytona = FakeDaytonaModule() a_builder(daytona=daytona).build(a_request()) From 13262094128da65adcf3cc00dbecfd2f18022967 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 10:24:24 +0200 Subject: [PATCH 20/22] release 1.9.20: a Daytona region Daytona knows, and a Dockerfile in the spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real Daytona build answered "Failed to create snapshot: Region not found": `region_id` was this platform's region, `r1`, which Daytona has never heard of. The region that scopes a snapshot is Daytona's own, named in `compatibility.regions`; with none named the field is left out and the account's default decides, which is what every snapshot in a real account has. `build.source: dockerfile` also gained somewhere to put the file — `build.dockerfile.content`, inline the way a requirements.txt already travels — so the contract and every capability report read it before anything is queued. All three managed variants take a Dockerfile now, each refusing what its own builder will not honour: E2B's list is read from its own parser, which prints "Unsupported instruction" and carries on, so a template would otherwise come back missing what the Dockerfile asked for and report success. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 9e8e7a4..339911f 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.19" +__version__ = "1.9.20" From 32bda6c408a45d5397c33869ca0cb28cb68e9724 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 12:36:29 +0200 Subject: [PATCH 21/22] attest: D-11 is the Datalayer artifact's, not every artifact's (E2-04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real Daytona build succeeded at the provider and then failed on our own side: "`51d10ab0-d98d-4117-bdb5-918e98646c92` is not a digest in a repository, so it cannot be attested". The snapshot was built, live in the owner's account, and the build was recorded as failed. `attest_artifact` demanded `registry/repository@sha256:…` of every variant. That is what a Datalayer artifact is: it lives in this platform's registry, the scanner reads it there, cosign signs that digest, and the Operator refuses to start what is unsigned. A managed artifact is none of those things — it lives in the owner's own provider account (D-8) and is named the way that provider names it: a Daytona snapshot uuid, an E2B build id, a Modal `im-…`. There is nothing in ECR to scan or sign, and no Operator starting it. So a managed variant records its artifact without a scan or a signature, and the digest check stays where it means something. Publishing is not weakened: E2-15's gate requires the *Datalayer* artifact to have passed its scan, and that one is still attested. The two tests that pinned the old behaviour asked it of `modal` and `e2b` — the variants this no longer applies to — and now ask `datalayer`. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/environments/attest.py | 31 +++++++++++- tests/test_environment_attest.py | 69 +++++++++++++++++++++------ 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/code_sandboxes/environments/attest.py b/code_sandboxes/environments/attest.py index 2db2d1c..0693d7a 100644 --- a/code_sandboxes/environments/attest.py +++ b/code_sandboxes/environments/attest.py @@ -675,14 +675,43 @@ def attest_artifact( answers the mapping the workflow stores: the scan's decision, the signature, the SBOM and provenance references, and the size. """ + variant = str(getattr(artifact, "variant", "") or "") reference = str(getattr(artifact, "immutable_reference", "") or "") + if variant and variant != "datalayer": + # D-11 is about the Datalayer artifact: it lives in this platform's + # registry, the scanner reads it there, cosign signs that digest, and + # the Operator refuses to start what is unsigned. A managed artifact + # is none of those things — it lives in the owner's own provider + # account (D-8) and is referenced the way that provider names it: a + # Daytona snapshot uuid, an E2B build id, a Modal `im-…`. There is no + # digest in a repository to scan or sign, and no Operator starting it. + # + # Attesting one anyway is what the first real Daytona build did, and + # it failed *after* the snapshot was built — the work done, the + # artifact live at the provider, and the build recorded as failed + # (2026-09-17). + # + # Publishing is not weakened by this: E2-15's gate requires the + # **Datalayer** artifact to have passed its scan, and that one is + # still attested here. + if log: + log(f"{variant} artifacts are not attested: {reference} is not a digest in a registry") + return { + "scan_summary": {}, + "sbom_ref": "", + "provenance_ref": "", + "signature_ref": "", + "size_bytes": size_bytes, + "signed_now": False, + "licenses": [], + } registry, _, rest = reference.partition("/") repository, _, digest = rest.partition("@") if not (registry and repository and _DIGEST.match(digest)): raise EnvironmentsError( PROVIDER_ERROR, f"`{reference}` is not a digest in a repository, so it cannot be attested", - detail={"reference": reference}, + detail={"reference": reference, "variant": variant}, ) use = attestor or Attestor( key=str(getattr(credential, "signing_key", "") or ""), diff --git a/tests/test_environment_attest.py b/tests/test_environment_attest.py index 44ce658..29641c0 100644 --- a/tests/test_environment_attest.py +++ b/tests/test_environment_attest.py @@ -19,12 +19,13 @@ import json import subprocess +from types import SimpleNamespace import pytest from code_sandboxes.environments.attest import Attestor, attest_artifact, licenses_of, signature_tag from code_sandboxes.environments.builders import ArtifactReference -from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.errors import PROVIDER_ERROR, EnvironmentsError from code_sandboxes.environments.policy import ( DEFAULT_POLICY, Finding, @@ -613,27 +614,25 @@ def test_a_blocked_artifact_is_never_signed(self) -> None: assert cosign.argv == [], "a blocked artifact must not be signed" def test_a_reference_that_is_not_a_digest_cannot_be_attested(self) -> None: - artifact = ArtifactReference( - variant="modal", - immutable_reference="im-1234567890", - provider_artifact_id="im-1234567890", - contract_version="sandbox-contract/v1", - ) + """A `datalayer` artifact must be a digest in this platform's registry. + + `attest_artifact` takes `artifact: Any`, so nothing guarantees every + caller went through the model validator that would have refused this. + """ + artifact = SimpleNamespace(variant="datalayer", immutable_reference="im-1234567890") with pytest.raises(EnvironmentsError) as raised: attest_artifact(artifact=artifact, attestor=an_attestor()) assert raised.value.code.code == "DL_ENV_PROVIDER_ERROR" def test_a_malformed_digest_cannot_be_attested_either(self) -> None: """`sha256:bad` starts with `sha256:` too: only a whole one is - accepted (found on PR #27's Copilot review). `e2b` rather than - `datalayer`, whose own model validator already refuses a malformed - digest before this function ever sees it — `attest_artifact` takes - `artifact: Any`, so nothing guarantees every caller went through it.""" - artifact = ArtifactReference( - variant="e2b", + accepted (found on PR #27's Copilot review). Asked of `datalayer`, + since that is the variant this check is for — `attest_artifact` takes + `artifact: Any`, so nothing guarantees every caller went through the + model validator that would have refused it.""" + artifact = SimpleNamespace( + variant="datalayer", immutable_reference=f"{REGISTRY}/{REPOSITORY}@sha256:bad", - provider_artifact_id="sha256:bad", - contract_version="sandbox-contract/v1", ) with pytest.raises(EnvironmentsError) as raised: attest_artifact(artifact=artifact, attestor=an_attestor()) @@ -749,3 +748,43 @@ def test_a_document_it_does_not_understand_names_nothing(self) -> None: """Never a reason to fail a build: a licence list is worth having, not dying for.""" for document in (None, {}, {"packages": None}, {"components": [1, 2]}, "spdx"): assert licenses_of(document) == [] + + +class TestWhatIsAttestedAndWhatIsNot: + """D-11 is about the Datalayer artifact, not every artifact. + + It lives in this platform's registry: the scanner reads it there, cosign + signs that digest, and the Operator refuses to start what is unsigned. A + managed artifact is none of those things — it lives in the owner's own + provider account (D-8), named the way that provider names it. + """ + + def _artifact(self, variant: str, reference: str): + return SimpleNamespace(variant=variant, immutable_reference=reference) + + def test_a_daytona_snapshot_is_not_attested(self) -> None: + """Its reference is a uuid, and attesting it anyway failed the first + real Daytona build *after* the snapshot was already built + (2026-09-17).""" + answer = attest_artifact( + artifact=self._artifact("daytona", "51d10ab0-d98d-4117-bdb5-918e98646c92"), + size_bytes=1234, + ) + assert answer["scan_summary"] == {} + assert answer["signature_ref"] == "" + assert answer["signed_now"] is False + # What the provider told us is still recorded. + assert answer["size_bytes"] == 1234 + + def test_an_e2b_build_id_and_a_modal_image_id_are_not_either(self) -> None: + for variant, reference in (("e2b", "bld-123"), ("modal", "im-abc123")): + answer = attest_artifact(artifact=self._artifact(variant, reference)) + assert answer["signed_now"] is False, variant + assert answer["scan_summary"] == {}, variant + + def test_a_datalayer_artifact_that_is_not_a_digest_still_fails(self) -> None: + """The check that matters is kept where it means something.""" + with pytest.raises(EnvironmentsError) as raised: + attest_artifact(artifact=self._artifact("datalayer", "not-a-digest")) + assert raised.value.code is PROVIDER_ERROR + assert "cannot be attested" in str(raised.value) From 0a3c1aa293321f26395332bcbe5fe04d831ff397 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Thu, 17 Sep 2026 12:46:03 +0200 Subject: [PATCH 22/22] release 1.9.21: a managed artifact is not attested as if it were ours The first real Daytona build succeeded at the provider and then failed on our own side, because `attest_artifact` demanded an OCI digest of every variant. The snapshot was built and live in the owner's account; the build was recorded as failed. D-11 is about the Datalayer artifact: it lives in this platform's registry, the scanner reads it there, cosign signs that digest, and the Operator refuses to start what is unsigned. A managed artifact lives in the owner's own provider account and is named the way that provider names it. Co-Authored-By: Claude Opus 5 (1M context) --- code_sandboxes/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 339911f..a1c4fc7 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.20" +__version__ = "1.9.21"