diff --git a/CHANGELOG.md b/CHANGELOG.md index c79771c..5a417bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,87 @@ ## Unreleased +## 1.9.18 + +- **A lock no longer carries a wall clock, so the build cache can hit** + (`environments/resolve.py`, `resolve_conda.py`; PLAN_ENVS.md E1-26, D-12). + The lock's header carried a `# resolved-at:` line, and its digest is over + the whole text — so two resolves of the same spec, in the same base, + pinning the same 320 packages, produced two different digests. Found by + resolving one environment twice on r1 on 2026-09-16: the texts differed in + exactly that one line out of 5,388. Section 5's cache key is over the lock + digest, so D-12's build cache could never hit, and it never had: + `environments.cache.lookups` read `hit=false` twelve times out of twelve. + When a lock was resolved is on the lock document Runtimes stores, in its + `created_at`, which is where it belongs. `resolved_at` is gone from + `lock_document`, `conda_lock_document`, `resolve_environment` and + `resolve_conda_environment`; the two tests that asserted determinism by + freezing the clock now assert it without one. + +## 1.9.17 + +- **An artifact's size is read from the registry** (`environments/attest.py`; + PLAN_ENVS.md E1-25). `attest_artifact` took `size_bytes` from its caller and + nobody ever passed one — the builder answers a reference, not a weight — so + every artefact was recorded with `sizeBytes: null` and + `environments.artifact.bytes`, the series section 14 tracks the artifact + size in, had no point in it although artifacts had been recorded (seen on r1, + 2026-09-16, through the OTEL query API). `Attestor.size_of()` asks the + registry, with the client the scan is already read from, and a size that + cannot be read is logged rather than raised: a missing number on a dashboard + is not a reason to refuse an artifact that is otherwise signed. 3 new tests. + +## 1.9.15 + +- **A restart restarts the kernel, not just this client's socket** + (`jupyter_server_sandbox`, `client`; PLAN_ENV.md E0-09, Appendix B check + 7). `CodeSandboxClient.restart()` was `stop()` then `start()`, which is + right for a sandbox this process owns — it is destroyed and recreated, and + nothing survives — and wrong for one *attached* to a Jupyter server + somebody else runs, which is every Datalayer runtime pod: stopping drops + the websocket while the kernel process keeps running, so the reconnect + lands in the same interpreter with every global still set. Check 7 is + "nothing is assumed to persist across restarts", and it read `state survived the restart ('True')` for exactly this reason — found live on r1, + 2026-09-16, the first drill whose smoke test reached the check. + `JupyterServerSandbox.restart_kernel()` now asks the server's own + `POST /api/kernels/{id}/restart` (the way `_do_interrupt` already uses the + API rather than the client's lifecycle) and reconnects onto the new + kernel; `restart()` prefers it and falls back to the lifecycle for every + variant that draws no such distinction. 7 new tests. +- **`datalayer/python-cpu:2026.09` repinned** to + `sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148`: + the contract layer now sets `MappingKernelManager.root_dir`, so kernels + start in `/home/datalayer/content`. The image already declared `WORKDIR` + there and `sandbox-contract/v1`'s User row already required it, but a + kernel's cwd is the Jupyter server's to choose and jupyter-python's config + roots it at `$HOME` — so every environment's kernel ran in + `/home/datalayer` and Appendix B check 2 read `cwd is '/home/datalayer', not '/home/datalayer/content'`. The file browser stays rooted at `$HOME`, + where a person expects to see everything they have; only the kernel moves. +- Two assertions that had rotted through three base releases are pinned in + one place again: the channel's digest and its apt snapshot were duplicated + across `test_environment_bases.py` and `test_environment_resolve.py`, and + 2026-09-15's and 2026-09-16's releases left both red rather than catching + anything. + +## 1.9.14 + +- **`datalayer/python-cpu:2026.09` base channel repinned** to the rebuilt + `jupyter-python:0.2.2` (now carrying `jupyter-kernels==1.2.23`) plus the + contract layer, digest + `sha256:aa5413000bb5b6ecd0a0cf03959b107f0d572f65bf230c08bbdf9a4569775545`, + released 2026-09-16 to `environments/base/python-cpu`. Every variant pins the + same digest. + +## 1.9.13 + +- **`jupyter-kernels==1.2.23` forced into `sandbox-contract/v1`**: it carries + the pooled kernel manager the runtime's Jupyter config selects + (`kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager`), + replacing the deprecated private `datalayer-kernels`. PyPI serves it, so a + resolve satisfies it from the index and the wheelhouse carries no wheel for + it; the pin keeps `uv pip sync --require-hashes` from stripping it out of a + user environment's image. + ## 1.9.12 - **`owner_repository`, `owner_cache_repository` and `ECR_ENVIRONMENT_PREFIX` diff --git a/code_sandboxes/__version__.py b/code_sandboxes/__version__.py index 1e71576..ed7b561 100644 --- a/code_sandboxes/__version__.py +++ b/code_sandboxes/__version__.py @@ -3,4 +3,4 @@ """Code Sandboxes.""" -__version__ = "1.9.12" +__version__ = "1.9.18" diff --git a/code_sandboxes/client.py b/code_sandboxes/client.py index 3e8105f..4ac2604 100644 --- a/code_sandboxes/client.py +++ b/code_sandboxes/client.py @@ -508,7 +508,21 @@ def is_alive(self) -> bool: return self.is_started def restart(self) -> None: - """Restart the wrapped sandbox through its public lifecycle.""" + """Restart the wrapped sandbox, clearing what it was holding. + + A sandbox this process owns is restarted by its own lifecycle: stop + and start destroy and recreate it, and nothing survives. One that is + merely *attached* to a server somebody else runs — a Jupyter server + in a Datalayer runtime pod — is not: stopping drops this client's + websocket while the kernel process goes on running, and starting + reconnects to the same interpreter with every global still set. A + sandbox that knows how to restart what it is attached to says so with + `restart_kernel`, and that is used in preference; the lifecycle is + the fallback for every variant that has no such distinction. + """ + restart_kernel = getattr(self._sandbox, "restart_kernel", None) + if callable(restart_kernel) and restart_kernel(): + return self._sandbox.stop() self._sandbox.start() diff --git a/code_sandboxes/environments/__init__.py b/code_sandboxes/environments/__init__.py index fa98607..f3cb16e 100644 --- a/code_sandboxes/environments/__init__.py +++ b/code_sandboxes/environments/__init__.py @@ -49,8 +49,12 @@ CONTRACT_V1, SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + BuildContextFinding, SandboxContract, + check_build_context, check_dockerfile, + validate_build_context, validate_dockerfile, ) from .errors import ERROR_CODES, EnvironmentsError, ErrorCode, map_provider_error @@ -91,6 +95,8 @@ "ApprovedBase", "ArtifactReference", "Attestor", + "BuildContextEntry", + "BuildContextFinding", "BuildRequest", "BuildkitResolveRunner", "CapabilityReport", @@ -114,6 +120,7 @@ "can_transition", "canonical_digest", "canonical_json", + "check_build_context", "check_dockerfile", "decide", "fingerprint_matches", @@ -131,6 +138,7 @@ "spec_digest", "spec_findings", "transition", + "validate_build_context", "validate_dockerfile", "validate_environment", ] diff --git a/code_sandboxes/environments/adapters/datalayer.py b/code_sandboxes/environments/adapters/datalayer.py index f3d82d8..49b085b 100644 --- a/code_sandboxes/environments/adapters/datalayer.py +++ b/code_sandboxes/environments/adapters/datalayer.py @@ -78,6 +78,12 @@ apt_snapshot_in, locked_versions, ) +from ..resolve_conda import ( + MICROMAMBA_BINARY, + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_dockerfile_line, +) from ..spec import BuildSecret, Environment, command_names_secret __all__ = [ @@ -229,8 +235,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca findings.append( CapabilityFinding( code=CAPABILITY_UNSUPPORTED.code, - message="conda environments are resolved by their own solver, " - "which is not built yet", + message="a conda environment is brought as a `dependencyFile` " + "whose `sourceFormat` is `conda`, not through `packages`", field="spec.packages.python.manager", ) ) @@ -253,7 +259,8 @@ def validate(self, environment: Environment, lock_text: str | None = None) -> Ca field="spec.platform.architecture", ) ) - if lock_text is not None and not locked_versions(lock_text): + pins_or_lock = lock_text is not None and not is_conda_lock(lock_text) + if pins_or_lock and not locked_versions(lock_text): findings.append( CapabilityFinding( code=SPEC_INVALID.code, @@ -323,18 +330,45 @@ def dockerfile(self, request: BuildRequest) -> str: f"COPY wheelhouse/ {imported_wheelhouse}/", 'RUN pip install --no-cache-dir "uv==0.12.11"', ] - lines.extend( - [ - "COPY lock.txt /opt/datalayer/lock.txt", - # `sync` and not `install`: the artifact holds the lock's set, - # and `--require-hashes` means every byte was the resolved one. - # `--find-links` for what no index has — a protected pin's - # own wheel, the fork's local version above all (E1-04). - "RUN --mount=type=cache,target=/root/.cache/uv " - f"uv pip sync --system --require-hashes --find-links {find_links} " - "/opt/datalayer/lock.txt", - ] - ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): the lock is an `@EXPLICIT` file + # `micromamba install --file` installs without re-solving, and the + # pip layer the solve resolved — the user's own pip requirements and + # the protected pins forced over them — is in the lock's own + # `# datalayer-pip:` header. The conda layer goes into the base's + # own environment; the pip layer follows, so the kernel stack + # (E1-04) and everything the solve installed is present the same as + # every source. micromamba is copied in from its pinned image + # first: the approved base bakes uv and the wheelhouse but not it. + pip_requirements = conda_lock_pip_requirements(request.lock_text) + lines.extend( + [ + micromamba_bootstrap_dockerfile_line(), + "COPY lock.txt /opt/datalayer/lock.txt", + "RUN --mount=type=cache,target=/opt/conda/pkgs " + f"{MICROMAMBA_BINARY} install --yes --name base " + "--file /opt/datalayer/lock.txt", + ] + ) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + lines.append( + "RUN --mount=type=cache,target=/root/.cache/uv " + f"uv pip install --system --find-links {find_links} {requirements}" + ) + else: + lines.extend( + [ + "COPY lock.txt /opt/datalayer/lock.txt", + # `sync` and not `install`: the artifact holds the lock's set, + # and `--require-hashes` means every byte was the resolved one. + # `--find-links` for what no index has — a protected pin's + # own wheel, the fork's local version above all (E1-04). + "RUN --mount=type=cache,target=/root/.cache/uv " + f"uv pip sync --system --require-hashes --find-links {find_links} " + "/opt/datalayer/lock.txt", + ] + ) # A build secret is mounted on the postInstall commands that name it # and nowhere else (§4.1, D-11): never an `ARG` or `ENV`, which bakes a # value into the image's history, never the package-install or files diff --git a/code_sandboxes/environments/adapters/daytona.py b/code_sandboxes/environments/adapters/daytona.py index 0a56c2c..f43dc1a 100644 --- a/code_sandboxes/environments/adapters/daytona.py +++ b/code_sandboxes/environments/adapters/daytona.py @@ -87,6 +87,7 @@ from __future__ import annotations +import shlex import tempfile import uuid from collections.abc import Callable @@ -110,6 +111,11 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, +) from ..spec import GPU_SIZE_CLASSES, Environment from .managed import ManagedBuilder @@ -171,6 +177,10 @@ class Builder(ManagedBuilder): variant = "daytona" item = "E2-04" title = "Daytona" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Daytona runs GPUs, on its own hardware and the owner's account (E2-17). #: This builder does not build one yet: see `_own_findings`. gpu = True @@ -343,28 +353,43 @@ def build(self, request: BuildRequest) -> ArtifactReference: # wheelhouse again would only duplicate what `uv pip sync` # can already reach at `WHEELHOUSE_IMAGE_PATH`. Only the # lock is genuinely per-build. - image = ( - image.add_local_file(str(lock_file), _LOCK_PATH) + image = image.add_local_file(str(lock_file), _LOCK_PATH) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` + # reads the `@EXPLICIT` lock without re-solving, and the + # pip layer the solve resolved — the user's pip + # requirements and the protected pins over them — comes + # from the lock's own `# datalayer-pip:` header, so the + # kernel stack (E1-04) and everything the solve installed is + # present the same as for a pip source. micromamba is + # installed first: the approved base bakes uv but not it. + image = image.run_commands(micromamba_bootstrap_command()) + image = image.run_commands( + f"micromamba install --yes --name base --file {_LOCK_PATH}" + ) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + image = image.run_commands( + "pip install --no-cache-dir " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {requirements}" + ) + else: # `uv` is not installed here: the approved base already # bakes it (E1-05, `resolve.py`'s own `bootstrap_uv` # docstring — "an approved Datalayer base already has it - # baked in"), and this phase's `build_sources` is - # `("packages",)` only, so every build starts from that - # base. Reinstalling it added an extra un-hashed network - # fetch outside the resolved lock for no reason (found in - # review) — matching the Datalayer builder's own - # `dockerfile()`, which installs `uv` only for the - # `image` source, not implemented for this variant yet. + # baked in"). Reinstalling it added an extra un-hashed + # network fetch outside the resolved lock for no reason + # (found in review). # # Packages install as root, the same reason the # Datalayer and E2B builders give: a user install lands # under the content directory's own home, which the # runtime mounts over. - .run_commands( + image = image.run_commands( "uv pip sync --system --require-hashes " f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}" ) - ) image = image.dockerfile_commands([f"USER 1000:100\nWORKDIR {_CONTENT_DIR}"]) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) diff --git a/code_sandboxes/environments/adapters/e2b.py b/code_sandboxes/environments/adapters/e2b.py index ce67135..875eca2 100644 --- a/code_sandboxes/environments/adapters/e2b.py +++ b/code_sandboxes/environments/adapters/e2b.py @@ -119,6 +119,7 @@ from __future__ import annotations +import shlex import tempfile from collections.abc import Callable from pathlib import Path @@ -140,6 +141,11 @@ ) from ..files import files_step from ..resolve import WHEELHOUSE_PATH, apt_pins_in +from ..resolve_conda import ( + conda_lock_pip_requirements, + is_conda_lock, + micromamba_bootstrap_command, +) from ..spec import Environment from .managed import ManagedBuilder @@ -191,6 +197,10 @@ class Builder(ManagedBuilder): variant = "e2b" item = "E2-03" title = "E2B" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Firecracker microVMs: no GPU passthrough. gpu = False #: E0-04's spike found only a registry login for the private base, never @@ -390,15 +400,38 @@ def build(self, request: BuildRequest) -> ArtifactReference: chain.copy("datalayer-sandbox", _DOCTOR_PATH, mode=0o755, user="root") .copy("wheelhouse", _WHEELHOUSE_PATH, user="root") .copy("lock.txt", _LOCK_PATH, user="root") - .run_cmd(f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root") - # Packages install as root (E0-04): a user install lands - # under /home/user, which the runtime mounts over. - .run_cmd( + ) + if is_conda_lock(request.lock_text): + # A conda source (E3-02): `micromamba install --file` reads the + # `@EXPLICIT` lock without re-solving, and the pip layer the + # solve resolved — the user's pip requirements and the protected + # pins over them — comes from the lock's own `# datalayer-pip:` + # header, so the kernel stack (E1-04) and everything the solve + # installed is present the same as for a pip source. micromamba + # is installed first: the approved base bakes uv but not it. + chain = chain.run_cmd(micromamba_bootstrap_command(), user="root") + chain = chain.run_cmd( + f"micromamba install --yes --name base --file {_LOCK_PATH}", + user="root", + ) + pip_requirements = conda_lock_pip_requirements(request.lock_text) + if pip_requirements: + requirements = " ".join(shlex.quote(req) for req in pip_requirements) + chain = chain.run_cmd( + f"pip install --no-cache-dir --find-links {_WHEELHOUSE_PATH} " + f"{requirements}", + user="root", + ) + else: + chain = chain.run_cmd( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', user="root" + ).run_cmd( + # Packages install as root (E0-04): a user install lands + # under /home/user, which the runtime mounts over. "uv pip sync --system --require-hashes " f"--find-links {_WHEELHOUSE_PATH} {_LOCK_PATH}", user="root", ) - ) for command in files_step(request.environment, variant=self.variant): chain = chain.run_cmd(command) for command in spec.commands.post_install: diff --git a/code_sandboxes/environments/adapters/managed.py b/code_sandboxes/environments/adapters/managed.py index a236e9c..668c4fc 100644 --- a/code_sandboxes/environments/adapters/managed.py +++ b/code_sandboxes/environments/adapters/managed.py @@ -36,7 +36,7 @@ ValidationResult, ) from ..errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError -from ..spec import GPU_SIZE_CLASSES, Environment +from ..spec import GPU_SIZE_CLASSES, Environment, EnvironmentSpec __all__ = ["ManagedBuilder"] @@ -67,6 +67,11 @@ class ManagedBuilder: supports_build_secrets = True #: The build sources it will accept in this phase. build_sources: tuple[str, ...] = ("packages",) + #: When `dependencyFile` is among `build_sources`, the `sourceFormat`s the + #: variant's own build actually installs. A conda source (E3-02) installs + #: with `micromamba`; a `pyproject`/`requirements` dependencyFile is not + #: built for a managed variant yet (E3-01), so it is not listed here. + dependency_formats: tuple[str, ...] = () package_managers: tuple[str, ...] = ("uv", "pip") #: Dockerfile instructions its own builder does not implement (§6). forbidden_instructions: tuple[str, ...] = () @@ -121,6 +126,8 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: field="spec.build.source", ) ) + elif spec.build.source == "dependencyFile": + findings.extend(self._dependency_file_findings(spec)) if spec.packages.python.manager not in self.package_managers: findings.append( CapabilityFinding( @@ -141,34 +148,7 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) ) if not self.gpu: - # A GPU is asked for two ways, and the spec's own validation - # couples them; a `validate` can be reached before that, so both - # are read here rather than trusting the coupling. - if spec.resources.size_class in GPU_SIZE_CLASSES: - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " - f"built for it. Drop {self.variant} from the variants, or build " - "the GPU classes for modal or daytona, which run them on their " - "own hardware" - ), - field="spec.resources.sizeClass", - ) - ) - elif spec.resources.accelerator != "none": - findings.append( - CapabilityFinding( - code=CAPABILITY_UNSUPPORTED.code, - message=( - f"{self.title} has no GPU, so an accelerator cannot be built for it. " - f"Drop {self.variant} from the variants, or build the GPU classes for " - "modal or daytona, which run them on their own hardware" - ), - field="spec.resources.accelerator", - ) - ) + findings.extend(self._gpu_findings(spec)) if self.regions: asked = [ region @@ -201,6 +181,58 @@ def _shared_findings(self, environment: Environment) -> list[CapabilityFinding]: ) return findings + def _gpu_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A variant with no GPU refuses either way a GPU is asked for. A GPU + is asked two ways, and the spec's own validation couples them; a + `validate` can be reached before that, so both are read here rather + than trusting the coupling.""" + if spec.resources.size_class in GPU_SIZE_CLASSES: + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so `{spec.resources.size_class}` cannot be " + f"built for it. Drop {self.variant} from the variants, or build " + "the GPU classes for modal or daytona, which run them on their " + "own hardware" + ), + field="spec.resources.sizeClass", + ) + ] + if spec.resources.accelerator != "none": + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"{self.title} has no GPU, so an accelerator cannot be built for it. " + f"Drop {self.variant} from the variants, or build the GPU classes for " + "modal or daytona, which run them on their own hardware" + ), + field="spec.resources.accelerator", + ) + ] + return [] + + def _dependency_file_findings(self, spec: EnvironmentSpec) -> list[CapabilityFinding]: + """A `dependencyFile` this variant accepts still only builds the + `sourceFormat`s it has an install step for (E3-01, E3-02): a conda + file installs with `micromamba`, but a `pyproject` or `requirements` + one is not built for a managed variant yet.""" + source_format = spec.build.dependency_file.source_format + if source_format in self.dependency_formats: + return [] + return [ + CapabilityFinding( + code=CAPABILITY_UNSUPPORTED.code, + message=( + f"a `{source_format}` dependency file is not built for " + f"{self.title} yet; it builds " + f"{', '.join(self.dependency_formats) or 'no dependency file'}" + ), + field="spec.build.dependencyFile.sourceFormat", + ) + ] + def _own_findings( self, environment: Environment, lock_text: str | None ) -> list[CapabilityFinding]: diff --git a/code_sandboxes/environments/adapters/modal.py b/code_sandboxes/environments/adapters/modal.py index 391e18f..abf6f40 100644 --- a/code_sandboxes/environments/adapters/modal.py +++ b/code_sandboxes/environments/adapters/modal.py @@ -147,6 +147,7 @@ from ..files import files_step from ..redact import redact from ..resolve import WHEELHOUSE_IMAGE_PATH, apt_pins_in +from ..resolve_conda import conda_lock_pip_requirements, is_conda_lock from ..spec import GPU_SIZE_CLASSES, BuildSecret, Environment, command_names_secret from .managed import ManagedBuilder @@ -211,6 +212,29 @@ def _scrubbed(text: str, values: dict[str, str]) -> str: return redact(text, values.values()) if values else text +def _install_packages(image: Any, lock_text: str) -> Any: + """The package layer for this lock: a conda source (E3-02) installs the + `@EXPLICIT` lock with Modal's own `micromamba_install` — which brings + micromamba itself, so no bootstrap is needed here — and layers the pip + layer the solve resolved (the user's pip requirements and the protected + pins over them, from the lock's own `# datalayer-pip:` header); a pip + source runs `uv pip sync`.""" + if is_conda_lock(lock_text): + image = image.micromamba_install(spec_file=_LOCK_PATH) + pip_requirements = conda_lock_pip_requirements(lock_text) + if pip_requirements: + image = image.pip_install(*pip_requirements, find_links=WHEELHOUSE_IMAGE_PATH) + return image + return image.run_commands( + f'pip install --no-cache-dir "uv=={_UV_VERSION}"', + # Packages install as root: every Modal build step already runs as + # root regardless of any `USER` line (see the module docstring), so + # this is stating what is already true rather than asking for it. + "uv pip sync --system --require-hashes " + f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", + ) + + def _post_install( image: Any, commands: list[str], declared: list[BuildSecret], step_secrets: dict[str, Any] ) -> Any: @@ -227,6 +251,10 @@ class Builder(ManagedBuilder): variant = "modal" item = "E2-05" title = "Modal" + #: A `packages` list and, for conda (E3-02), an `environment.yml` + #: dependency file installed with `micromamba_install`. + build_sources = ("packages", "dependencyFile") + dependency_formats = ("conda",) #: Modal runs GPUs, in the owner's workspace (E2-17). This builder does #: not build one yet: see `build`'s own guard. gpu = True @@ -403,15 +431,7 @@ def build(self, request: BuildRequest) -> ArtifactReference: image = image.add_local_file( str(entrypoint_file), _ENTRYPOINT_PATH, copy=True ).run_commands(f"chmod +x {_ENTRYPOINT_PATH}") - image = image.run_commands( - f'pip install --no-cache-dir "uv=={_UV_VERSION}"', - # Packages install as root: every Modal build step - # already runs as root regardless of any `USER` line - # (see the module docstring), so this is stating what - # is already true rather than asking for it. - "uv pip sync --system --require-hashes " - f"--find-links {WHEELHOUSE_IMAGE_PATH} {_LOCK_PATH}", - ) + image = _install_packages(image, request.lock_text) for command in files_step(request.environment, variant=self.variant): image = image.run_commands(command) image = _post_install(image, spec.commands.post_install, declared, step_secrets) diff --git a/code_sandboxes/environments/attest.py b/code_sandboxes/environments/attest.py index 94c750d..fcf7d88 100644 --- a/code_sandboxes/environments/attest.py +++ b/code_sandboxes/environments/attest.py @@ -536,10 +536,35 @@ def attest( signature_ref=signature, sbom_ref=sbom_ref or f"{registry}/{repository}@{digest}.sbom", provenance_ref=provenance_ref or f"{registry}/{repository}@{digest}.att", - size_bytes=size_bytes, + size_bytes=size_bytes + if size_bytes is not None + else self.size_of(repository=repository, digest=digest), signed_now=signed_now, ) + def size_of(self, *, repository: str, digest: str) -> int | None: + """What the registry says the artifact weighs, or None. + + Nobody hands the size down: the builder answers a reference, not a + weight, so until this asked the registry `size_bytes` was always None + — and with it `environments.artifact.bytes`, the series section 14 + tracks the artifact size in, which had no point in it on 2026-09-16 + although artifacts had been recorded. The registry has known all + along; the scan is read from the same client. + + Never a reason to fail an attestation: a size that could not be read + is a missing number on a dashboard, and the artifact is still signed. + """ + try: + images = self._client().describe_images( + repositoryName=repository, imageIds=[{"imageDigest": digest}] + )["imageDetails"] + except Exception as error: + self._log(f"The artifact's size could not be read: {error}") + return None + size = (images[0] or {}).get("imageSizeInBytes") if images else None + return int(size) if size else None + def _client(self) -> Any: if self._ecr is None: try: diff --git a/code_sandboxes/environments/bases.py b/code_sandboxes/environments/bases.py index 252ab1a..a374e7c 100644 --- a/code_sandboxes/environments/bases.py +++ b/code_sandboxes/environments/bases.py @@ -109,8 +109,9 @@ def repository(self) -> str: base.ref: base for base in ( # E1-05: jupyter-python:0.2.2 (Ubuntu security packages and conda's own - # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped) plus the - # contract layer, released 2026-09-14 to environments/base/python-cpu. + # OpenSSL upgraded, JupyterLab's staging yarn.lock dropped, jupyter-kernels + # installed) plus the contract layer, released 2026-09-16 to + # environments/base/python-cpu. # One image, so every variant pins the same digest until a variant # needs a base of its own. The prior digest, jupyter-python:0.2.1, # carried 31 fixable-critical findings the scan (E1-08) blocks every @@ -125,10 +126,19 @@ def repository(self) -> str: # `.spec.VARIANTS`, spelled out: `spec` imports from this # module, so importing it back here would be circular. ("datalayer", "e2b", "daytona", "modal"), - "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36", + # 2026-09-16: kernels now start in the contract's own + # working directory. The image already declared `WORKDIR + # /home/datalayer/content`, but a kernel's cwd is the + # Jupyter server's to choose, and jupyter-python's config + # roots it at `$HOME` — so every environment's kernel ran + # in `/home/datalayer` and Appendix B check 2 read "cwd is + # '/home/datalayer', not '/home/datalayer/content'". The + # contract layer now sets `MappingKernelManager.root_dir`, + # which moves the kernel without moving the file browser. + "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148", ) }, - snapshots={"2026.09": "20260914T150000Z"}, + snapshots={"2026.09": "20260916T120000Z"}, ), # E2-17: jupyter-python-cuda plus the same layer. ApprovedBase( diff --git a/code_sandboxes/environments/conformance.py b/code_sandboxes/environments/conformance.py index 200f6d6..0447f68 100644 --- a/code_sandboxes/environments/conformance.py +++ b/code_sandboxes/environments/conformance.py @@ -546,8 +546,13 @@ def _gpu(sandbox: Sandbox, requested: bool, cuda: str | None, timeout: float | N problems.append("no GPU is visible") if cuda and not str(answer.get("cuda") or "").startswith(cuda): problems.append(f"CUDA is {answer.get('cuda')}, not {cuda}") + # A GPU version gates on this (E2-17): a version that asked for an + # accelerator and cannot see it, or sees the wrong CUDA, is not the + # version its spec describes. A version that asked for none never + # reaches here (the trivial pass above), so the extended tier still + # gates nothing for a CPU version. return _result( - 11, not problems, gating=False, detail="; ".join(problems) or None, actual=answer + 11, not problems, gating=True, detail="; ".join(problems) or None, actual=answer ) @@ -625,10 +630,17 @@ def run_extended_tier( concurrent_kernels: int = 4, timeout: float | None = 120.0, ) -> ValidationResult: - """Appendix B checks 10-14: recorded per variant, gating nothing.""" + """Appendix B checks 10-14: recorded per variant. Only check 11 gates, and + only for a version that asked for an accelerator (E2-17) — a GPU version + that cannot see its GPU is not what its spec describes; every other + extended check records without gating.""" checks = [ _guard(10, False, lambda: _egress(sandbox, egress_allowed, egress_blocked, timeout)), - _guard(11, False, lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout)), + _guard( + 11, + accelerator_requested, + lambda: _gpu(sandbox, accelerator_requested, cuda_version, timeout), + ), _guard(12, False, lambda: _throughput(sandbox, contract, minimum_mib_per_second, timeout)), _guard(13, False, lambda: _cold_start(cold_start_seconds, cold_start_budget)), _guard(14, False, lambda: _concurrent(sandbox, concurrent_kernels, timeout)), @@ -649,7 +661,10 @@ def run_conformance( extended: Mapping[str, Any] | None = None, timeout: float | None = 120.0, ) -> ValidationResult: - """Both tiers: the core tier decides, the extended tier is recorded beside it.""" + """Both tiers together. The core tier decides; the extended tier is recorded + beside it, save for check 11, which also decides for a version that asked for + an accelerator (E2-17) — a GPU version that cannot see its GPU has not built + what its spec described.""" core = run_core_tier( sandbox, python_version=python_version, diff --git a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt index 3f27ac5..d5afa7c 100644 --- a/code_sandboxes/environments/constraints/sandbox-contract-v1.txt +++ b/code_sandboxes/environments/constraints/sandbox-contract-v1.txt @@ -20,8 +20,18 @@ # jupyterlab and jupyter-server-ydoc require jupyter-server>=2.19. The rebased fork # satisfies those ranges, so pip keeps it; its local label, which PyPI never serves, # tells it apart from the release. +# +# jupyter-kernels carries the pooled kernel manager the runtime's Jupyter config +# selects (kernel_manager_class = jupyter_kernels.pool.mapping.PooledMappingKernelManager, +# plane/etc/dockerfiles/jupyter-python/etc/jupyter/config/jupyter_config.py). Unlike the +# deprecated private datalayer-kernels it replaced, PyPI serves it, so a resolve satisfies +# it from the index and the wheelhouse carries no wheel for it. The pin is still forced in: +# without it `uv pip sync --require-hashes` strips it from a user environment's image and the +# runtime's Jupyter server crashloops on a kernel_manager_class it can no longer import +# (found live on 2026-09-15, the first user-environment build to reach the smoke test). ipykernel==7.3.0 jupyter-client==8.9.1 jupyter-server==2.21.0+datalayer.1 jupyter-server-nbmodel==0.2.8 +jupyter-kernels==1.2.23 datalayer==1.7.4 diff --git a/code_sandboxes/environments/constraints/wheelhouse/README.md b/code_sandboxes/environments/constraints/wheelhouse/README.md index bbf2e79..2ed7334 100644 --- a/code_sandboxes/environments/constraints/wheelhouse/README.md +++ b/code_sandboxes/environments/constraints/wheelhouse/README.md @@ -38,3 +38,10 @@ Pure Python (`py3-none-any`), so one wheel serves every base's Python version this channel carries. Rebuild it here, under this same file name, whenever `services/kernels/Dockerfile`'s pinned commit changes — the constraints file's own version pin and this wheel move together. + +The pooled kernel manager pin, `jupyter-kernels`, needs no wheel here: PyPI +serves it, so a resolve satisfies it from the index like any other pin. It +was previously the private `datalayer-kernels`, which no index carried and +so was baked here as a wheel; the migration to the public +[`jupyter-kernels`](https://pypi.org/project/jupyter-kernels/) package +dropped that wheel. diff --git a/code_sandboxes/environments/contract.py b/code_sandboxes/environments/contract.py index 592c3c7..8df1fe8 100644 --- a/code_sandboxes/environments/contract.py +++ b/code_sandboxes/environments/contract.py @@ -24,26 +24,34 @@ import re import shlex import sys +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from pydantic import BaseModel, ConfigDict from .bases import is_approved_repository -from .errors import CAPABILITY_UNSUPPORTED, EnvironmentsError +from .errors import CAPABILITY_UNSUPPORTED, SPEC_INVALID, EnvironmentsError __all__ = [ "CONTRACT_V1", + "MAX_CONTEXT_FILES", + "MAX_CONTEXT_FILE_BYTES", + "MAX_CONTEXT_TOTAL_BYTES", "SANDBOX_CONTRACT_V1", "SUPPORTED_CONTRACTS", + "BuildContextEntry", + "BuildContextFinding", "ContractRow", "DockerfileFinding", "DockerfileInstruction", "SandboxContract", + "check_build_context", "check_dockerfile", "contract_markdown", "get_contract", "parse_dockerfile", + "validate_build_context", "validate_dockerfile", ] @@ -453,6 +461,94 @@ def check_dockerfile(text: str, *, contract: SandboxContract = SANDBOX_CONTRACT_ ) +# --- The build context (E3-03) ------------------------------------------------------- + +#: A `dockerfile` source uploads a build context to object storage. These bound +#: what Runtimes accepts before it issues a presigned URL, so a context cannot +#: be a way to smuggle a host file in (a symlink or `..`), or to fill a bucket. +MAX_CONTEXT_FILES = 2000 +MAX_CONTEXT_FILE_BYTES = 50 * 1024 * 1024 +MAX_CONTEXT_TOTAL_BYTES = 100 * 1024 * 1024 + +_CONTEXT_SEPARATOR = re.compile(r"[\\/]") + + +@dataclass(frozen=True) +class BuildContextEntry: + """One member of an uploaded build context: its path, size, and whether it is a symlink.""" + + path: str + size_bytes: int = 0 + is_symlink: bool = False + + +@dataclass(frozen=True) +class BuildContextFinding: + """Something in a build context that must not be uploaded.""" + + path: str + message: str + + def to_dict(self) -> dict[str, object]: + return {"path": self.path, "message": self.message} + + +def validate_build_context( + entries: Sequence[BuildContextEntry], +) -> list[BuildContextFinding]: + """Everything in a build context the upload refuses, in the order given. + + Refused: an absolute path, a `..` that would escape the context, a symlink + (which could point at a host file the build then reads), a file over the + per-file limit, and — once — a context with too many files or too many + bytes in all. + """ + findings: list[BuildContextFinding] = [] + total = 0 + for entry in entries: + path = entry.path + components = _CONTEXT_SEPARATOR.split(path) + if not path or all(part in ("", ".") for part in components): + findings.append(BuildContextFinding(path, "is not a path inside the context")) + elif path.startswith("/") or path.startswith("\\"): + findings.append(BuildContextFinding(path, "is an absolute path, not a context path")) + elif ".." in components: + findings.append(BuildContextFinding(path, "escapes the context with `..`")) + if entry.is_symlink: + findings.append( + BuildContextFinding(path, "is a symlink, which could read a host file") + ) + if entry.size_bytes > MAX_CONTEXT_FILE_BYTES: + findings.append( + BuildContextFinding( + path, f"is over the {MAX_CONTEXT_FILE_BYTES}-byte per-file limit" + ) + ) + total += entry.size_bytes + if len(entries) > MAX_CONTEXT_FILES: + findings.append( + BuildContextFinding("", f"has more than {MAX_CONTEXT_FILES} files") + ) + if total > MAX_CONTEXT_TOTAL_BYTES: + findings.append( + BuildContextFinding("", f"is over the {MAX_CONTEXT_TOTAL_BYTES}-byte total limit") + ) + return findings + + +def check_build_context(entries: Sequence[BuildContextEntry]) -> None: + """Refuse a build context the upload does not allow, naming the first fault.""" + findings = validate_build_context(entries) + if findings: + first = findings[0] + where = f"`{first.path}`: " if first.path else "" + raise EnvironmentsError( + SPEC_INVALID, + f"{where}{first.message}", + detail={"findings": [finding.to_dict() for finding in findings]}, + ) + + # --- The documentation page ---------------------------------------------------------- diff --git a/code_sandboxes/environments/files.py b/code_sandboxes/environments/files.py index e31cac6..e26f516 100644 --- a/code_sandboxes/environments/files.py +++ b/code_sandboxes/environments/files.py @@ -25,12 +25,15 @@ def build_entries( environment: Environment, *, source_of: Callable[[FileEntry], str] | None = None ) -> list[BuildEntry]: - """The spec's files as build entries. + """The spec's baked files as build entries: ``files`` then ``contentsBuild``. - ``source_of`` turns a ``contentRef`` into a URL the build can fetch — - a presigned URL for a ``blob://`` reference — and defaults to the - reference itself. A file without its sha256 is refused: the build - verifies every byte it bakes. + ``files`` a user uploaded (referenced by ``contentRef``) and + ``contentsBuild`` fetched from an external ``source`` are both baked the + same way, so they are the same kind of entry here. ``source_of`` turns a + ``contentRef`` into a URL the build can fetch — a presigned URL for a + ``blob://`` reference — and defaults to the reference itself; a + ``contentsBuild`` source is already a URL and is used as is. A file without + its sha256 is refused: the build verifies every byte it bakes. """ entries: list[BuildEntry] = [] for index, entry in enumerate(environment.spec.files): @@ -48,6 +51,15 @@ def build_entries( size_bytes=entry.size_bytes, ) ) + for built in environment.spec.contents_build: + entries.append( + BuildEntry( + source_uri=built.source, + destination_path=built.path, + sha256=built.sha256, + size_bytes=built.size_bytes, + ) + ) return entries @@ -63,7 +75,7 @@ def files_step( """ if variant not in VARIANTS: raise ValueError(f"{variant!r} is not a variant") - if not environment.spec.files: + if not environment.spec.files and not environment.spec.contents_build: return [] build = EnvironmentBuild( environment=environment.metadata.name, diff --git a/code_sandboxes/environments/resolve.py b/code_sandboxes/environments/resolve.py index 811d65c..16b9f04 100644 --- a/code_sandboxes/environments/resolve.py +++ b/code_sandboxes/environments/resolve.py @@ -49,7 +49,7 @@ from datetime import datetime, timezone from functools import lru_cache from pathlib import Path -from typing import Any, Callable, Protocol +from typing import TYPE_CHECKING, Any, Callable, Protocol from .bases import APPROVED_BASES, ApprovedBase, channel_snapshot, resolve_base from .build_secrets import resolve_build_secret @@ -76,6 +76,9 @@ parse_requirements_txt, ) +if TYPE_CHECKING: + from .resolve_conda import CondaResolveRunner + __all__ = [ "APT_PIN_PREFIX", "APT_SNAPSHOT_PREFIX", @@ -886,7 +889,6 @@ def lock_document( python_version: str, base_reference: str, merged: MergedRequirements, - resolved_at: datetime | None = None, ) -> dict[str, Any]: """The stored lock: its text, its digest, and what a reader needs from it. @@ -894,11 +896,19 @@ def lock_document( output, so the document is still a requirements file — ``pip install -r`` reads it, and so does every tool that only knows that format — while saying everything the build installs. + + **Nothing here is a wall clock.** The header carried a ``# resolved-at:`` + line until 2026-09-16, and since the digest is over the whole text, two + resolves of the same spec, in the same base, pinning the same 320 packages + produced two different digests — the texts differed in that one line out + of 5,388, found by resolving the same environment twice on r1. The cache + key of section 5 is over the lock digest, so D-12's build cache could + never hit: `environments.cache.lookups` read `hit=false` 12 times out of + 12. When the lock was resolved is on the lock document Runtimes stores, in + its ``created_at``, where it belongs. """ - when = (resolved_at or _utcnow()).replace(microsecond=0).isoformat() header = [ "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", - f"# resolved-at: {when}", f"# python: {python_version}", f"# base: {base_reference}", ] @@ -1158,8 +1168,8 @@ def resolve_environment( credential: Any = None, log: Callable[[str], None] | None = None, runner: ResolveRunner | None = None, + conda_runner: CondaResolveRunner | None = None, bases: dict[str, ApprovedBase] = APPROVED_BASES, - resolved_at: datetime | None = None, uv: str | None = None, pyproject_run: Callable[..., subprocess.CompletedProcess[str]] | None = None, image_transport: Any = None, @@ -1185,11 +1195,15 @@ def resolve_environment( Where the solve's output goes, line by line: the build's log. runner Where the solve runs. D-9's BuildKit solve by default. + conda_runner + Where a conda ``dependencyFile``'s own ``micromamba`` solve runs + (E3-02): the conda seam's runner, injected by tests and by ``plane + local`` the same way ``runner`` is for a pip source, and D-9's BuildKit + conda solve by default. A pip source ignores it, and a conda source + ignores ``runner``, since the two solves are different tools. bases The approved bases, injected by tests and by a plane whose channel is published somewhere else. - resolved_at - The moment the lock records. Now by default. uv, pyproject_run A `pyproject` `dependencyFile` source's own verification (E3-01): the `uv` to check and export with, and how it is run — injected by @@ -1252,6 +1266,19 @@ def resolve_environment( ) dependency_file = environment.spec.build.dependency_file if source == "dependencyFile" and dependency_file is not None: + if dependency_file.source_format == "conda": + # A conda `environment.yml` resolves through its own micromamba + # solve into an explicit lock (E3-02), not uv's pip compile. + from .resolve_conda import resolve_conda_environment + + return resolve_conda_environment( + environment_yml=dependency_file.content, + python_version=environment.spec.language.version, + resolved_bases=resolved_bases, + credential=credential, + log=say, + runner=conda_runner, + ) if dependency_file.source_format == "pyproject": # Verified, not re-resolved (E3-01): the author's own uv.lock is # the answer, and this only proves it still matches pyproject.toml. @@ -1297,7 +1324,6 @@ def resolve_environment( python_version=environment.spec.language.version, base_reference=solving_in, merged=merged, - resolved_at=resolved_at, ) say( f"Locked {document['package_count']} packages" diff --git a/code_sandboxes/environments/resolve_conda.py b/code_sandboxes/environments/resolve_conda.py new file mode 100644 index 0000000..e6462dd --- /dev/null +++ b/code_sandboxes/environments/resolve_conda.py @@ -0,0 +1,912 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an ``environment.yml`` to one explicit lock (E3-02). + +A ``build.source: dependencyFile`` version whose ``sourceFormat`` is ``conda`` +brings a conda ``environment.yml`` — the form advanced users already have for a +project that pulls, say, ``gdal`` from ``conda-forge`` rather than from a wheel. +It is resolved the way every other source is (PLAN_ENV.md §5, D-9): **once**, +into one lock, and every variant of the version then builds from that one lock, +so four sandboxes of the same version carry the same packages. + +Two decisions this item makes, both recorded here rather than left implicit: + +- **micromamba, not conda-lock.** The solve runs ``micromamba`` — the same tool + the Datalayer, E2B and Daytona builders install the lock with, and the tool + Modal's ``micromamba_install`` wraps — so the interpreter that resolves is the + one that installs, with no second solver's opinion in between. The lock it + produces is a conda **explicit** file (``@EXPLICIT``): one ``package-url#hash`` + line per package, which ``micromamba create --file`` installs without + re-solving. That is the "explicit lock with hashes" this box asks for. +- **The protected pins still apply to the pip layer.** A conda environment needs + the same kernel stack every sandbox needs to connect (``ipykernel`` and its + siblings, ``constraints/sandbox-contract-v1.txt``). They are merged *over* the + ``pip:`` section of the ``environment.yml`` exactly as :func:`merge_requirements` + merges them over a ``packages`` source's dependencies — a pip requirement that + contradicts a protected pin is refused with the range that is supported, and + every pin is forced in whether or not the environment named it, because a + ``--constraint`` alone never installs what nothing else already depends on. + +Where the solve runs is the :class:`CondaResolveRunner`'s business, mirroring +:mod:`code_sandboxes.environments.resolve`: + +- :class:`BuildkitCondaResolveRunner` is D-9's: a BuildKit solve ``FROM`` the + resolved base digest, which gives resolution the builder's isolation and its + egress allowlist. +- :class:`MicromambaResolveRunner` runs ``micromamba`` where it is called, for + ``plane local`` and for tests. + +@module code_sandboxes.environments.resolve_conda +""" + +from __future__ import annotations + +import hashlib +import re +import shlex +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Protocol + +from .errors import ( + CAPABILITY_UNSUPPORTED, + PACKAGE_NOT_FOUND, + PROVIDER_ERROR, + RESOLVE_CONFLICT, + SPEC_INVALID, + EnvironmentsError, +) +from .resolve import ( + WHEELHOUSE_IMAGE_PATH, + WHEELHOUSE_PATH, + MergedRequirements, + ProtectedPin, + merge_requirements, +) + +__all__ = [ + "CONDA_LOCK_FORMAT", + "BuildkitCondaResolveRunner", + "CondaEnvironment", + "CondaResolveOutcome", + "CondaResolveRequest", + "CondaResolveRunner", + "MicromambaResolveRunner", + "conda_lock_document", + "conda_lock_pip_requirements", + "explicit_lock_packages", + "is_conda_lock", + "merge_conda_pip", + "parse_conda_environment", + "parse_conda_failure", + "resolve_conda_environment", +] + +#: What a conda lock is, as the version records it: a conda **explicit** file, +#: the ``@EXPLICIT`` form ``micromamba create --file`` installs without solving. +CONDA_LOCK_FORMAT = "conda-explicit" + +#: The platform a Phase-2 artifact is built for (D-9): every variant is +#: ``linux/amd64``, so the solve is for ``linux-64`` in conda's own naming. +CONDA_PLATFORM = "linux-64" + +#: The marker the ``@EXPLICIT`` conda lock body opens with, and the key the +#: ``environment.yml`` names its pip layer under. +_EXPLICIT_MARKER = "@EXPLICIT" +_PIP_SECTION_KEY = "pip" + +#: How the whole pip layer is recorded in the lock's header — the user's pip +#: requirements *and* the protected pins Datalayer forces over them, pinned to +#: the versions the solve resolved — one ``# datalayer-pip: `` line each, +#: above the ``@EXPLICIT`` body. A builder installs the conda layer from the +#: body and then this pip layer, so the whole of what a version resolved to is +#: in the one document and nothing resolved in the solve is lost from the build. +CONDA_PIP_PREFIX = "# datalayer-pip: " + +#: The ``micromamba`` the conda solve and every conda build use, pinned so the +#: tool that resolves is the tool that installs (E3-02): the approved base bakes +#: uv, the wheelhouse and the doctor, but not micromamba, so it is brought in +#: here rather than assumed. A BuildKit build copies the binary from this image; +#: a builder driving an SDK installs the same pinned release. +MICROMAMBA_VERSION = "2.0.5" +MICROMAMBA_IMAGE = f"mambaorg/micromamba:{MICROMAMBA_VERSION}" +MICROMAMBA_BINARY = "/usr/local/bin/micromamba" + +#: A channel URL that carries a credential in its userinfo — the same shape +#: :func:`code_sandboxes.environments.spec._index_findings` refuses in an index +#: URL, so a token is caught the same way whichever field names it. +_URL_CREDENTIALS = re.compile(r"^[a-z][a-z0-9+.-]*://[^/@\s]+:[^/@\s]*@", re.IGNORECASE) + + +def micromamba_bootstrap_dockerfile_line() -> str: + """The Dockerfile line that brings the pinned micromamba into a build. + + ``COPY --from`` the pinned micromamba image, so the binary is present and + reproducible without a network fetch inside the build itself. Used by the + resolver's own solve image and by the Datalayer (BuildKit) builder. + """ + return f"COPY --from={MICROMAMBA_IMAGE} /bin/micromamba {MICROMAMBA_BINARY}" + + +def micromamba_bootstrap_command() -> str: + """The shell command that installs the pinned micromamba into a build. + + For a builder that drives an SDK (E2B, Daytona) rather than emitting a + Dockerfile: the same pinned release ``COPY --from`` brings, fetched into + ``/usr/local/bin`` so a later ``micromamba install`` finds it on the PATH. + """ + return ( + f"curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/{MICROMAMBA_VERSION} " + "| tar -xj -C /usr/local/bin --strip-components=1 bin/micromamba" + ) + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +# -- Reading the environment.yml ---------------------------------------------- + + +@dataclass(frozen=True) +class CondaEnvironment: + """An ``environment.yml``, as the resolver reads it.""" + + name: str + channels: tuple[str, ...] + conda_dependencies: tuple[str, ...] + pip_dependencies: tuple[str, ...] + + +def parse_conda_environment(text: str) -> CondaEnvironment: + """One ``environment.yml``'s channels, conda packages and pip packages. + + A conda ``dependencies`` list mixes plain conda specs with a single + ``{"pip": [...]}`` mapping for the pip layer; both are separated here so the + protected pins merge over the pip layer alone (:func:`merge_conda_pip`) and + the conda layer is passed through untouched. Anything that is not a conda + spec or the one pip mapping — a nested list, a bare number — is refused with + its position, because a solve is not the place to discover a malformed file. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError as error: + raise EnvironmentsError( + SPEC_INVALID, + f"the environment.yml is not valid YAML: {error}", + detail={"field": "spec.build.dependencyFile.content"}, + ) from error + if not isinstance(document, Mapping): + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml is empty or not a mapping", + detail={"field": "spec.build.dependencyFile.content"}, + ) + raw_dependencies = document.get("dependencies") + if raw_dependencies is None: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names no `dependencies`", + detail={"field": "spec.build.dependencyFile.content"}, + ) + if not isinstance(raw_dependencies, Sequence) or isinstance(raw_dependencies, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`dependencies` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.dependencies"}, + ) + conda: list[str] = [] + pip: list[str] = [] + seen_pip = False + for index, entry in enumerate(raw_dependencies): + field_name = f"spec.build.dependencyFile.content.dependencies[{index}]" + if isinstance(entry, str): + spec = entry.strip() + if spec: + conda.append(spec) + continue + if isinstance(entry, Mapping) and set(entry) == {_PIP_SECTION_KEY}: + if seen_pip: + raise EnvironmentsError( + SPEC_INVALID, + "the environment.yml names more than one `pip:` section", + detail={"field": field_name}, + ) + seen_pip = True + pip.extend(_pip_requirements(entry[_PIP_SECTION_KEY], field_name)) + continue + raise EnvironmentsError( + SPEC_INVALID, + "a `dependencies` entry in the environment.yml is neither a conda " + "spec nor a single `pip:` section", + detail={"field": field_name}, + ) + channels = _channels(document) + return CondaEnvironment( + name=str(document.get("name") or "environment"), + channels=channels, + conda_dependencies=tuple(conda), + pip_dependencies=tuple(pip), + ) + + +def _pip_requirements(entries: Any, field_name: str) -> list[str]: + """The strings of a ``pip:`` section, or a refusal naming its position.""" + if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "the `pip:` section in the environment.yml is not a list", + detail={"field": f"{field_name}.pip"}, + ) + requirements: list[str] = [] + for pip_index, requirement in enumerate(entries): + if not isinstance(requirement, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `pip:` entry in the environment.yml is not a string", + detail={"field": f"{field_name}.pip[{pip_index}]"}, + ) + spec = requirement.strip() + if spec: + requirements.append(spec) + return requirements + + +def _channels(document: Mapping[str, Any]) -> tuple[str, ...]: + raw = document.get("channels") + if raw is None: + return () + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)): + raise EnvironmentsError( + SPEC_INVALID, + "`channels` in the environment.yml is not a list", + detail={"field": "spec.build.dependencyFile.content.channels"}, + ) + channels: list[str] = [] + for index, channel in enumerate(raw): + if not isinstance(channel, str): + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry in the environment.yml is not a string", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) + text = channel.strip() + if text: + if _URL_CREDENTIALS.match(text): + # The same refusal `spec.packages.python.indexes` gives a + # credential-bearing index URL: a token in the channel is + # copied into the solve context and can reappear in the + # explicit lock, so it belongs in the build's secrets, not here. + raise EnvironmentsError( + SPEC_INVALID, + "a `channels` entry carries a credential in its URL; reference the " + "credential in `buildSecrets`", + detail={"field": f"spec.build.dependencyFile.content.channels[{index}]"}, + ) + channels.append(text) + return tuple(channels) + + +def merge_conda_pip( + environment: CondaEnvironment, + pins: Sequence[ProtectedPin] | None = None, +) -> MergedRequirements: + """Datalayer's protected pins merged over the environment's ``pip:`` layer. + + The conda layer is Datalayer's to leave alone — the kernel stack is a pip + concern — but the pip layer is merged exactly as a ``packages`` source's + dependencies are (:func:`merge_requirements`): a pip requirement that + contradicts a protected pin is refused, and every pin is forced in whether + or not the environment named it. The one that has no index — the + jupyter-server fork wheel (E1-04) — is satisfied from the wheelhouse the + same ``--find-links`` reaches in the pip step below. + """ + return merge_requirements(environment.pip_dependencies, pins=pins) + + +def rendered_environment( + environment: CondaEnvironment, + merged: MergedRequirements, + *, + python_version: str, +) -> str: + """The ``environment.yml`` the solve is actually given. + + The user's conda packages and channels, with two things settled: the + interpreter is pinned to the base's ``python`` (D-9, never silently + replaced), and the ``pip:`` section is the merged one — the user's pip + requirements with the protected pins forced over them. + """ + import yaml + + conda_without_python = [ + spec for spec in environment.conda_dependencies if _conda_package_name(spec) != "python" + ] + dependencies: list[Any] = [f"python={python_version}", *conda_without_python] + pip_layer = list(merged.requirements) + if pip_layer: + dependencies.append({_PIP_SECTION_KEY: pip_layer}) + document = { + "name": environment.name, + "channels": list(environment.channels) or ["conda-forge"], + "dependencies": dependencies, + } + return yaml.safe_dump(document, sort_keys=False, default_flow_style=False) + + +def _conda_package_name(spec: str) -> str: + """The package a conda spec names, lowercased: ``python`` from + ``python=3.13``, ``python >=3.11`` or ``python[version='3.13']``.""" + return re.split(r"[\s=<>!~\[]", spec.strip(), maxsplit=1)[0].strip().lower() + + +# -- What a runner is asked, and what it answers ------------------------------ + + +@dataclass(frozen=True) +class CondaResolveRequest: + """One conda solve: what to resolve, for which interpreter, from where.""" + + environment_yml: str + """The rendered ``environment.yml`` — interpreter pinned, pins merged.""" + + python_version: str + platform: str = CONDA_PLATFORM + base_reference: str = "" + """The base the solve runs inside, pinned by digest (D-9).""" + + registry_auth: Mapping[str, str] | None = None + """What the registry needs to be read, when the runner pulls the base.""" + + +@dataclass +class CondaResolveOutcome: + """A conda solve's answer: the explicit lock, and the pip layer it resolved. + + ``lock_text`` is the ``@EXPLICIT`` conda lock, verbatim from ``micromamba``. + ``pip_lock`` is the pip layer the same solve installed — the user's pip + requirements and the protected pins forced over them — pinned to the + versions it resolved, read back from ``micromamba env export`` so the + artifact carries the whole of what the solve produced, not the conda layer + alone (E3-02). A runner that cannot read the prefix back leaves it empty, + and :func:`conda_lock_document` falls back to the merged requirements. + """ + + lock_text: str + pip_lock: tuple[str, ...] = () + + +def pip_requirements_from_env_yaml(text: str) -> tuple[str, ...]: + """The pip layer a ``micromamba env export`` names, pinned, in order. + + A conda ``env export`` (the YAML form, not ``--explicit``) lists the pip + packages it installed under a single ``{"pip": [...]}`` entry of its + ``dependencies``, each ``name==version`` — cleanly separated from the conda + packages, which are their own strings. This reads that section back, so the + solve's resolved pip versions become the lock's pip layer. A malformed or + pip-less export is an empty layer, never a raised error: the explicit lock + is what a solve is judged by, and its own marker is checked elsewhere. + """ + import yaml + + try: + document = yaml.safe_load(text) + except yaml.YAMLError: + return () + if not isinstance(document, Mapping): + return () + dependencies = document.get("dependencies") + if not isinstance(dependencies, Sequence) or isinstance(dependencies, (str, bytes)): + return () + requirements: list[str] = [] + for entry in dependencies: + if isinstance(entry, Mapping) and _PIP_SECTION_KEY in entry: + for requirement in entry[_PIP_SECTION_KEY] or []: + if isinstance(requirement, str) and requirement.strip(): + requirements.append(requirement.strip()) + return tuple(requirements) + + +class CondaResolveRunner(Protocol): + """Where a conda solve runs.""" + + name: str + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: ... + + +# -- Reading micromamba's refusals -------------------------------------------- + +_NOTHING_PROVIDES = re.compile( + r"nothing provides (?:requested )?(?P[A-Za-z0-9][A-Za-z0-9._-]*)" +) +_PACKAGE_NOT_FOUND = re.compile( + r"(?:package|libmamba).*?(?P[A-Za-z0-9][A-Za-z0-9._-]*) is not available" +) + + +def parse_conda_failure(output: str) -> EnvironmentsError: + """``micromamba``'s refusal as one of section 10's codes. + + A package no channel serves, and an unsatisfiable set of packages, are the + version's own to fix and are reported as such. Anything else — the channel + unreachable, ``micromamba`` missing — is ``DL_ENV_PROVIDER_ERROR``, which is + retryable, because nothing about the version is wrong. + """ + text = " ".join(line.strip() for line in output.splitlines() if line.strip()) + detail: dict[str, Any] = {"output": text[:2000]} + + provides = _NOTHING_PROVIDES.search(text) + if provides: + name = provides.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + unavailable = _PACKAGE_NOT_FOUND.search(text) + if unavailable: + name = unavailable.group("name") + return EnvironmentsError( + PACKAGE_NOT_FOUND, + f"`{name}` is not in any channel this environment may read", + detail={**detail, "package": name}, + ) + lowered = text.lower() + if ( + "could not solve" in lowered + or "unsolvable" in lowered + or "encountered problems while solving" in lowered + or "no solution" in lowered + ): + return EnvironmentsError( + RESOLVE_CONFLICT, + "The conda dependencies cannot be satisfied together: " + text[:400], + detail=detail, + ) + return EnvironmentsError( + PROVIDER_ERROR, + "The conda resolver did not finish: " + text[:400], + detail=detail, + ) + + +# -- Running the solve -------------------------------------------------------- + + +class MicromambaResolveRunner: + """``micromamba`` where this runs: for ``plane local`` and for tests. + + Creates the environment the ``environment.yml`` asks for into a scratch + prefix and exports it as an explicit lock. The prefix is thrown away; only + the lock is kept, which is the one thing every variant then builds from. + """ + + name = "micromamba" + + def __init__(self, micromamba: str | None = None, timeout: float = 900.0) -> None: + # `None` means "find it"; an empty string means "there is none", which + # is how a test says so without hiding micromamba from the process. + self._micromamba = (shutil.which("micromamba") or "") if micromamba is None else micromamba + self._timeout = timeout + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._micromamba: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `micromamba` to resolve with: the local conda resolver needs it on the PATH", + detail={"missing": "micromamba", "runner": self.name}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-") as directory: + root = Path(directory) + spec_file = root / "environment.yml" + spec_file.write_text(request.environment_yml, encoding="utf-8") + prefix = root / "prefix" + create = [ + self._micromamba, + "create", + "--yes", + "--prefix", + str(prefix), + "--platform", + request.platform, + "--file", + str(spec_file), + ] + say(f"Resolving the conda environment for {request.platform}") + try: + created = subprocess.run( # noqa: S603 - the argv is built here + create, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env={**_os_environ(), "PIP_FIND_LINKS": str(WHEELHOUSE_PATH)}, + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if created.returncode != 0: + for line in (created.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(created.stderr or created.stdout or "") + export = self._export( + [self._micromamba, "env", "export", "--explicit", "--prefix", str(prefix)], + say, + ) + # The pip layer the same solve installed, pinned, read back from the + # YAML export's own `pip:` section (E3-02): the explicit export above + # carries conda packages alone, so without this the user's resolved + # pip requirements would be absent from the artifact. + pip_export = self._export( + [self._micromamba, "env", "export", "--prefix", str(prefix)], + say, + ) + return CondaResolveOutcome( + lock_text=export.stdout, + pip_lock=pip_requirements_from_env_yaml(pip_export.stdout), + ) + + def _export( + self, argv: list[str], say: Callable[[str], None] + ) -> subprocess.CompletedProcess[str]: + """One ``micromamba env export``, its timeout handled the same as the solve. + + The ``create`` above and both exports share the one refusal so a timeout + anywhere becomes ``DL_ENV_PROVIDER_ERROR`` rather than a raw + :class:`subprocess.TimeoutExpired` a caller cannot classify or retry. + """ + try: + result = subprocess.run( # noqa: S603 - the argv is built here + argv, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + if result.returncode != 0: + for line in (result.stderr or "").splitlines(): + say(line) + raise parse_conda_failure(result.stderr or result.stdout or "") + return result + + +class BuildkitCondaResolveRunner: + """D-9's conda solve: ``micromamba`` inside the resolved base, under BuildKit. + + The solve gets the builder's isolation, its egress allowlist and the exact + interpreter the artifact will have. It needs ``buildctl`` and a reachable + ``buildkitd``, neither of which exists until the build pool is deployed + (E1-06), so until then it refuses by name rather than resolving elsewhere. + """ + + name = "buildkit-conda" + + def __init__( + self, + buildctl: str | None = None, + address: str | None = None, + tlscert: str | None = None, + tlskey: str | None = None, + tlscacert: str | None = None, + timeout: float = 1200.0, + ) -> None: + self._buildctl = (shutil.which("buildctl") or "") if buildctl is None else buildctl + self._tlscert = tlscert or "" + self._tlskey = tlskey or "" + self._tlscacert = tlscacert or "" + self._address = address or "" + self._timeout = timeout + + def _tls_options(self) -> list[str]: + if not (self._tlscert and self._tlskey and self._tlscacert): + return [] + return [ + f"--tlscert={self._tlscert}", + f"--tlskey={self._tlskey}", + f"--tlscacert={self._tlscacert}", + ] + + def dockerfile(self, request: CondaResolveRequest) -> str: + """The solve, as the frontend reads it: create the env, then export it. + + The ``environment.yml`` is a spec field a user wrote, so its path is the + only thing that reaches the ``RUN`` line — its content is a file copied + into the context, never interpolated into a shell command — and the + wheelhouse is brought along for the one protected pin no index has + (E1-04), reached through ``PIP_FIND_LINKS`` the same way the local + runner reaches it. ``micromamba`` is copied in from its pinned image + first, because the approved base bakes uv and the wheelhouse but not it. + The explicit lock and the pip layer are both exported, so the artifact + carries the whole of what the solve resolved, not the conda layer alone. + """ + micromamba = shlex.quote(MICROMAMBA_BINARY) + return ( + "\n".join( + [ + f"FROM {request.base_reference} AS solve", + "USER root", + "WORKDIR /solve", + micromamba_bootstrap_dockerfile_line(), + "COPY environment.yml ./environment.yml", + "ENV PIP_FIND_LINKS=" + shlex.quote(WHEELHOUSE_IMAGE_PATH), + "RUN --mount=type=cache,target=/opt/conda/pkgs " + f"{micromamba} create --yes --prefix /solve/prefix " + f"--platform {shlex.quote(request.platform)} --file environment.yml", + f"RUN {micromamba} env export --explicit " + "--prefix /solve/prefix > /solve/lock.txt", + f"RUN {micromamba} env export --prefix /solve/prefix > /solve/pip-env.yml", + "FROM scratch", + "COPY --from=solve /solve/lock.txt /lock.txt", + "COPY --from=solve /solve/pip-env.yml /pip-env.yml", + ] + ) + + "\n" + ) + + def solve( + self, request: CondaResolveRequest, log: Callable[[str], None] | None = None + ) -> CondaResolveOutcome: + say = log or (lambda _line: None) + if not self._buildctl: + raise EnvironmentsError( + CAPABILITY_UNSUPPORTED, + "No `buildctl` to resolve with: the build pool is not deployed here", + detail={"missing": "buildctl", "runner": self.name, "item": "E1-06"}, + ) + if "@sha256:" not in request.base_reference: + raise EnvironmentsError( + SPEC_INVALID, + "The base is resolved to a digest before anything is solved", + detail={"base": request.base_reference}, + ) + with tempfile.TemporaryDirectory(prefix="dl-conda-solve-") as directory: + root = Path(directory) + (root / "environment.yml").write_text(request.environment_yml, encoding="utf-8") + (root / "Dockerfile").write_text(self.dockerfile(request), encoding="utf-8") + out = root / "out" + command = [ + self._buildctl, + *(["--addr", self._address] if self._address else []), + *self._tls_options(), + "build", + "--frontend", + "dockerfile.v0", + "--local", + f"context={root}", + "--local", + f"dockerfile={root}", + "--output", + f"type=local,dest={out}", + ] + say(f"Solving the conda lock in {request.base_reference}") + try: + finished = subprocess.run( # noqa: S603 - the argv is built here + command, + capture_output=True, + text=True, + timeout=self._timeout, + check=False, + env=self._environment(request), + ) + except subprocess.TimeoutExpired as expired: + raise EnvironmentsError( + PROVIDER_ERROR, + f"The conda solve did not finish within {self._timeout:.0f}s", + detail={"runner": self.name, "timeout": self._timeout}, + ) from expired + for line in (finished.stderr or "").splitlines(): + say(line) + if finished.returncode != 0: + raise parse_conda_failure(finished.stderr or finished.stdout or "") + lock = (out / "lock.txt").read_text(encoding="utf-8") + pip_env = out / "pip-env.yml" + pip_lock = ( + pip_requirements_from_env_yaml(pip_env.read_text(encoding="utf-8")) + if pip_env.exists() + else () + ) + return CondaResolveOutcome(lock_text=lock, pip_lock=pip_lock) + + def _environment(self, request: CondaResolveRequest) -> dict[str, str] | None: + auth = dict(request.registry_auth or {}) + if not auth: + return None + return {**_os_environ(), **{str(key): str(value) for key, value in auth.items()}} + + +def _os_environ() -> dict[str, str]: + import os + + return dict(os.environ) + + +# -- The lock ----------------------------------------------------------------- + + +def explicit_lock_packages(lock_text: str) -> list[str]: + """Every package an explicit lock installs, one URL per line. + + An ``@EXPLICIT`` file is comments, the ``@EXPLICIT`` marker, and then one + ``https://…/pkg.conda#hash`` line per package; the URLs are what a build + installs and what this counts. + """ + packages: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or line == _EXPLICIT_MARKER: + continue + packages.append(line) + return packages + + +def is_conda_lock(lock_text: str | None) -> bool: + """Whether a lock is a conda explicit lock, and not the pip one. + + A conda lock carries the ``@EXPLICIT`` marker; a pip lock never does. A + builder reads this to install with ``micromamba`` rather than ``uv pip + sync`` — the one signal that travels with the lock text itself, so a + builder handed only :attr:`BuildRequest.lock_text` still knows which it is. + """ + if not lock_text: + return False + return any(line.strip() == _EXPLICIT_MARKER for line in lock_text.splitlines()) + + +def conda_lock_pip_requirements(lock_text: str) -> list[str]: + """The whole pip layer a conda lock's header records, in order. + + :func:`conda_lock_document` writes the pip layer the solve resolved as + ``# datalayer-pip: `` lines above the ``@EXPLICIT`` body: the user's + own pip requirements and the protected pins Datalayer forced over them, + pinned to the versions the solve produced. A builder installs the conda + layer from the body and then this pip layer, so the whole of what the + version resolved to is built and nothing the solve installed is lost (E3-02). + """ + prefix = CONDA_PIP_PREFIX.strip() + requirements: list[str] = [] + for raw in lock_text.splitlines(): + line = raw.strip() + if line.startswith(prefix): + requirement = line[len(prefix) :].strip() + if requirement: + requirements.append(requirement) + return requirements + + +def conda_lock_document( + outcome: CondaResolveOutcome, + *, + python_version: str, + base_reference: str, + merged: MergedRequirements, + platform: str = CONDA_PLATFORM, +) -> dict[str, Any]: + """The stored conda lock: its text, its digest, and what a reader needs. + + The pip layer the solve resolved is written as comments above the explicit + lock — the ``# datalayer-pip:`` lines a builder installs after the conda + packages — so the one document says the whole of what a build installs: the + conda packages by URL and hash, and the user's pip requirements with the + protected pins Datalayer forced over them, pinned to the versions the solve + produced. It stays a file ``micromamba create --file`` reads unchanged. + The pip layer is the solve's own (``outcome.pip_lock``) when the runner + could read the prefix back, and the merged requirements otherwise, so it is + always complete rather than the protected pins alone. + """ + header = [ + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.", + f"# python: {python_version}", + f"# platform: {platform}", + f"# base: {base_reference}", + ] + pip_layer = list(outcome.pip_lock) or list(merged.requirements) + for requirement in pip_layer: + header.append(f"{CONDA_PIP_PREFIX}{requirement}") + body = outcome.lock_text.lstrip("\n") + if _EXPLICIT_MARKER not in {line.strip() for line in body.splitlines()}: + raise EnvironmentsError( + PROVIDER_ERROR, + "micromamba did not produce an explicit lock (no @EXPLICIT marker)", + detail={"format": CONDA_LOCK_FORMAT}, + ) + text = "\n".join(header) + "\n" + body + if not text.endswith("\n"): + text += "\n" + packages = explicit_lock_packages(text) + return { + "digest": "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest(), + "format": CONDA_LOCK_FORMAT, + "content": text, + "python_version": python_version, + "package_count": len(packages), + } + + +def resolve_conda_environment( + *, + environment_yml: str, + python_version: str, + resolved_bases: Mapping[str, str], + platform: str = CONDA_PLATFORM, + credential: Any = None, + log: Callable[[str], None] | None = None, + runner: CondaResolveRunner | None = None, +) -> dict[str, Any]: + """A conda version's lock, from its ``environment.yml`` (E3-02). + + This is the conda seam of :func:`code_sandboxes.environments.resolve.resolve_environment`: + it takes the ``environment.yml`` a ``dependencyFile`` source carries and the + bases already resolved for the wanted variants, and answers the same lock + document every other source answers — ``digest``, ``format``, ``content``, + ``python_version``, ``package_count`` — so the workflow stores it the same way. + + Raises + ------ + EnvironmentsError + Everything a person can act on: a malformed ``environment.yml``, a pip + requirement that contradicts a protected pin, a conflict, or a package + no channel serves. + """ + say = log or (lambda _line: None) + environment = parse_conda_environment(environment_yml) + merged = merge_conda_pip(environment) + for note in merged.notes: + say(note) + rendered = rendered_environment(environment, merged, python_version=python_version) + solving_in = resolved_bases.get("datalayer") or next(iter(resolved_bases.values())) + request = CondaResolveRequest( + environment_yml=rendered, + python_version=python_version, + platform=platform, + base_reference=solving_in, + registry_auth=_registry_auth(credential), + ) + outcome = (runner or BuildkitCondaResolveRunner()).solve(request, say) + document = conda_lock_document( + outcome, + python_version=python_version, + base_reference=solving_in, + merged=merged, + platform=platform, + ) + say(f"Locked {document['package_count']} conda packages as {document['digest']}") + return {**document, "resolved_bases": dict(resolved_bases)} + + +def _registry_auth(credential: Any) -> Mapping[str, str] | None: + """The credential's registry auth, however it carries it — as + :func:`code_sandboxes.environments.resolve._registry_auth` reads it, kept + in step so both resolvers accept the one credential shape durable mints.""" + if credential is None: + return None + for attribute in ("registry_auth", "environment", "env"): + value = getattr(credential, attribute, None) + if callable(value): + value = value() + if isinstance(value, Mapping): + return {str(key): str(item) for key, item in value.items()} + return None diff --git a/code_sandboxes/environments/spec.py b/code_sandboxes/environments/spec.py index f0d8f3c..d180f9a 100644 --- a/code_sandboxes/environments/spec.py +++ b/code_sandboxes/environments/spec.py @@ -58,6 +58,7 @@ "SIZE_CLASSES", "SUPPORTED_BUILD_SOURCES", "VARIANTS", + "PUBLIC_PACKAGE_INDEX_HOSTS", "Accelerator", "ArtifactStatus", "Base", @@ -84,6 +85,7 @@ "VersionStatus", "assert_publishable", "command_names_secret", + "index_is_public", "parse_environment", "parse_requirements_txt", "publication_findings", @@ -105,7 +107,7 @@ BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") #: What builds today; `dockerfile` is the one source still to come. -SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "image") +SUPPORTED_BUILD_SOURCES: tuple[str, ...] = ("packages", "dependencyFile", "dockerfile", "image") SUPPORTED_PACKAGE_MANAGERS: tuple[str, ...] = ("uv", "pip") #: `requirements.txt` and `pyproject.toml`/`uv.lock` are archived on the #: version they resolved (E3-01); this bounds what a spec may carry inline, @@ -167,6 +169,72 @@ class Platform(_Model): architecture: Literal["linux/amd64"] = "linux/amd64" +#: The package indexes D-12 counts as public: a version may be published only +#: when every index it resolves from is one of these, since a private index is +#: reached with a credential the public does not hold. Matched on host, so the +#: trailing `/simple` or its absence never decides it. `pypi.org` is the index; +#: `files.pythonhosted.org` is where its wheels are served from. +PUBLIC_PACKAGE_INDEX_HOSTS = frozenset( + {"pypi.org", "files.pythonhosted.org"} +) + + +def _package_index_host(url: str) -> str: + """The host an index URL names, lower-cased and without its port, or `""`.""" + from urllib.parse import urlsplit # noqa: PLC0415 + + try: + return (urlsplit(url).hostname or "").lower() + except ValueError: + return "" + + +def index_is_public(url: str) -> bool: + """Whether an index URL is one D-12 lets a published version resolve from.""" + return _package_index_host(url) in PUBLIC_PACKAGE_INDEX_HOSTS + + +#: The conda channels D-12 counts as public: a published conda version +#: (E3-02's ``dependencyFile``) may resolve only from these, since a private +#: channel is reached with a token no public reader holds — the same boundary +#: :data:`PUBLIC_PACKAGE_INDEX_HOSTS` draws for pip indexes. The bare names +#: anaconda.org serves openly, and the hosts a channel URL may name; any other +#: name or host is private and blocks publication. +PUBLIC_CONDA_CHANNELS = frozenset( + { + "conda-forge", + "bioconda", + "defaults", + "nodefaults", + "main", + "r", + "anaconda", + "pkgs/main", + "pkgs/r", + "msys2", + } +) +PUBLIC_CONDA_CHANNEL_HOSTS = frozenset( + {"conda.anaconda.org", "repo.anaconda.com", "anaconda.org"} +) + + +def channel_is_public(channel: str) -> bool: + """Whether a conda channel is one D-12 lets a published version resolve from. + + A channel is a URL, whose host must be a public conda host, or a bare name, + which is public only when it is one of the well-known open channels — an + unlisted name (say a private org's) is treated as private, since a bare name + on anaconda.org may still need a token the public does not have. + """ + text = channel.strip() + if not text: + return True + if "://" in text: + return _package_index_host(text) in PUBLIC_CONDA_CHANNEL_HOSTS + return text.lower() in PUBLIC_CONDA_CHANNELS + + class PythonPackages(_Model): manager: Literal["uv", "pip", "conda"] = "uv" dependencies: list[str] = Field(default_factory=list) @@ -191,6 +259,21 @@ class FileEntry(_Model): sha256: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class ContentsBuildEntry(_Model): + """One immutable file the Environment bakes from an external source. + + Unlike ``files`` — which a user uploads, referenced by ``contentRef`` — a + ``contentsBuild`` entry names an external ``source`` URL fetched at build + time and verified against ``sha256`` (required: the build verifies every + byte it bakes). Both are baked the same way, into every provider artifact. + """ + + source: str + path: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + size_bytes: int | None = Field(default=None, ge=0) + + class Commands(_Model): post_install: list[str] = Field(default_factory=list) @@ -242,17 +325,20 @@ class Compatibility(_Model): class DependencyFileSpec(_Model): - """A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01). + """A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda + `environment.yml` (E3-01, E3-02). ``requirements`` resolves the way ``packages`` does — the protected constraints merged in, the same solve. ``pyproject`` does not resolve at all: its own ``uv.lock`` is verified against the current ``pyproject.toml`` and exported, never re-solved, because a lock the - author already made is the whole point of bringing one. + author already made is the whole point of bringing one. ``conda`` resolves + the ``environment.yml`` in its own ``micromamba`` solve into an explicit + lock, with the protected constraints merged over its ``pip:`` layer. """ - source_format: Literal["requirements", "pyproject"] = "requirements" - #: The `requirements.txt` text, or the `pyproject.toml` text. + source_format: Literal["requirements", "pyproject", "conda"] = "requirements" + #: The `requirements.txt`, `pyproject.toml` or conda `environment.yml` text. content: str = "" #: The `uv.lock` text. Required, and only meaningful, for `pyproject`. lock_content: str = "" @@ -294,6 +380,7 @@ class EnvironmentSpec(_Model): platform: Platform = Field(default_factory=Platform) packages: Packages = Field(default_factory=Packages) files: list[FileEntry] = Field(default_factory=list) + contents_build: list[ContentsBuildEntry] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) commands: Commands = Field(default_factory=Commands) build_secrets: list[BuildSecret] = Field(default_factory=list) @@ -409,7 +496,11 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings: list[SpecFinding] = [] if not dependency_file.content.strip(): name = ( - "pyproject.toml" if dependency_file.source_format == "pyproject" else "requirements.txt" + "pyproject.toml" + if dependency_file.source_format == "pyproject" + else "environment.yml" + if dependency_file.source_format == "conda" + else "requirements.txt" ) findings.append(SpecFinding(f"{field}.content", f"is empty; it is the {name} text")) elif dependency_file.source_format == "requirements": @@ -419,6 +510,8 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding(f"{field}.content[{index}]", f"`{requirement}`: {problem}") ) + elif dependency_file.source_format == "conda": + findings.extend(_conda_environment_findings(dependency_file.content)) if len(dependency_file.content.encode("utf-8")) > MAX_DEPENDENCY_FILE_BYTES: findings.append( SpecFinding(f"{field}.content", f"is over {MAX_DEPENDENCY_FILE_BYTES} bytes") @@ -438,12 +531,31 @@ def _dependency_file_findings(dependency_file: DependencyFileSpec | None) -> lis findings.append( SpecFinding( f"{field}.lockContent", - "is only read for a `pyproject` source; a `requirements` source resolves fresh", + "is only read for a `pyproject` source; a `requirements` or `conda` source " + "resolves fresh", ) ) return findings +def _conda_environment_findings(content: str) -> list[SpecFinding]: + """An `environment.yml`'s own shape, before it reaches the conda solver (E3-02). + + The same validate-before-resolve rule every source follows: a malformed + `environment.yml` is the version's to fix, and refusing it here — with the + field the resolver would have named — is cheaper than a solve that fails on + it after taking a worker. + """ + from .resolve_conda import parse_conda_environment + + field = "spec.build.dependencyFile.content" + try: + parse_conda_environment(content) + except EnvironmentsError as error: + return [SpecFinding(error.detail.get("field", field), error.message, error.code)] + return [] + + def _image_findings(image: ImageSourceSpec | None) -> list[SpecFinding]: field = "spec.build.image" if image is None: @@ -497,10 +609,12 @@ def spec_findings( ) ) - # An `image` source brings its own base (E3-04): `spec.base` names - # nothing Datalayer approved, so checking it against the table would - # refuse every import for the one reason imports exist to avoid. - if spec.build.source != "image": + # An `image` source brings its own base (E3-04), and a `dockerfile` + # source's base is the `FROM` its uploaded Dockerfile names (E3-03, + # validated against the approved bases by `check_dockerfile`, not here): + # `spec.base` names nothing Datalayer approved for either, so checking it + # against the table would refuse every one for the reason they exist. + if spec.build.source not in ("image", "dockerfile"): base = bases.get(spec.base.ref) if base is None: findings.append( @@ -810,17 +924,15 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: D-12: *"A promoted version becomes public only by being published, and only when every input is public — public indexes, no ``files``, no - ``buildSecrets``, an approved base."* This function holds only the - ``buildSecrets`` half of that boundary — a build secret is IAM-held and - fetched for one build's own use, so it is never public by definition, - whether the version is otherwise made of nothing but public inputs or - not. The rest of D-12's boundary (public indexes, no baked ``files``, an - approved base) belongs to the publish route itself once it exists - (E2-15, not built yet as of this writing — there is no - ``services/library`` "environment" artifact type and no publish endpoint - in ``services/runtimes/datalayer_runtimes/services/environments.py`` to - call this from today). This is the seam that route calls when it lands, - named the way every other rule of the specification is. + ``buildSecrets``, an approved base."* This function holds the input half + of that boundary that the spec alone decides against a credential the + public does not hold: a build secret is IAM-held and so never public, and + a private package index is reached with a credential no public reader has. + The parts of D-12 that depend on a version's *status* rather than its + spec — a passing scan, a signed artifact, and that the datalayer variant's + base is an approved one — are the publish route's own to check against the + artifact it publishes (E2-15), since ``publication_findings`` is handed + the spec and nothing built from it. Deliberately never applied to `promote()` (the private, per-owner lifecycle step that makes a version an environment's active one): a @@ -828,17 +940,70 @@ def publication_findings(environment: Environment) -> list[SpecFinding]: ever builds or launches it (D-12's own words). Only the act of making a version world-visible is refused. """ - if not environment.spec.build_secrets: - return [] - ids = ", ".join(secret.id for secret in environment.spec.build_secrets) - return [ - SpecFinding( - "spec.buildSecrets", - f"a version with a build secret ({ids}) can never be published to the " - "public Library (D-12); remove it, or keep the version private", - PUBLICATION_BLOCKED, + findings: list[SpecFinding] = [] + if environment.spec.build_secrets: + ids = ", ".join(secret.id for secret in environment.spec.build_secrets) + findings.append( + SpecFinding( + "spec.buildSecrets", + f"a version with a build secret ({ids}) can never be published to the " + "public Library (D-12); remove it, or keep the version private", + PUBLICATION_BLOCKED, + ) + ) + private = [ + url + for url in environment.spec.packages.python.indexes + if not index_is_public(url) + ] + if private: + findings.append( + SpecFinding( + "spec.packages.python.indexes", + f"a version that resolves from a private index ({', '.join(private)}) " + "can never be published to the public Library (D-12); publish only from " + f"public indexes ({', '.join(sorted(PUBLIC_PACKAGE_INDEX_HOSTS))})", + PUBLICATION_BLOCKED, + ) ) + private_channels = [ + channel for channel in _conda_channels(environment) if not channel_is_public(channel) ] + if private_channels: + findings.append( + SpecFinding( + "spec.build.dependencyFile.content.channels", + "a version that resolves from a private conda channel " + f"({', '.join(private_channels)}) can never be published to the public " + "Library (D-12); publish only from public channels " + f"({', '.join(sorted(PUBLIC_CONDA_CHANNELS))})", + PUBLICATION_BLOCKED, + ) + ) + return findings + + +def _conda_channels(environment: Environment) -> tuple[str, ...]: + """The channels a conda ``dependencyFile`` names, or none for any other source. + + A conda ``environment.yml``'s ``channels`` are package inputs the same as a + pip source's indexes, so publication weighs them the same (D-12). A file + that will not parse has no channels to weigh here — validation refuses it + before it is ever published — so a parse failure is an empty tuple, not a + raise. + """ + build = environment.spec.build + dependency_file = build.dependency_file + if build.source != "dependencyFile" or dependency_file is None: + return () + if dependency_file.source_format != "conda": + return () + from .resolve_conda import parse_conda_environment + + try: + return parse_conda_environment(dependency_file.content).channels + except EnvironmentsError: + return () def assert_publishable(environment: Environment) -> None: diff --git a/code_sandboxes/jupyter_server_sandbox.py b/code_sandboxes/jupyter_server_sandbox.py index cbb0aec..a729f69 100644 --- a/code_sandboxes/jupyter_server_sandbox.py +++ b/code_sandboxes/jupyter_server_sandbox.py @@ -607,6 +607,58 @@ def _do_interrupt(self) -> bool: logger.warning(f"Failed to interrupt Jupyter kernel: {e}") return False + def restart_kernel(self) -> bool: + """Restart the kernel itself, through the server's own REST API. + + Not `stop()` then `start()`: those are this *client's* lifecycle, and + when the server is somebody else's — a Datalayer runtime pod, which + is every attached sandbox — stopping the client only drops the + websocket. The kernel is a process on the server and keeps running, + so a reconnect lands back in the same interpreter with every global + still set. + + That is what Appendix B check 7 ("nothing is assumed to persist + across restarts") measures, and it read `state survived the restart` + for exactly this reason — found live on r1, 2026-09-16, the first + drill whose smoke test reached the check. `POST + /api/kernels/{id}/restart` is the one that restarts the kernel, the + same way `_do_interrupt` already uses the API rather than the + client's own lifecycle. + + Answers whether the server accepted it; never raises, so a caller + that cannot restart reports a failed check rather than an error. + """ + if not self._server_url or not self._client: + return False + kernel_id = getattr(self._client, "id", None) + if not kernel_id: + return False + try: + response = requests.post( + f"{self._server_url}/api/kernels/{kernel_id}/restart", + params={"token": self._token}, + headers=self._headers or None, + timeout=30, + ) + except Exception as error: + # A restart that cannot even be asked for is a failed check, not + # an error to raise at the caller, exactly as `_do_interrupt` is. + logger.warning(f"Failed to restart Jupyter kernel: {error}") + return False + if not response.ok: + logger.warning( + "Failed to restart Jupyter kernel: the server answered %s", response.status_code + ) + return False + # The websocket the client holds is to the kernel that has just been + # replaced; reconnecting is what makes the next execution land in the + # new interpreter rather than on a channel nobody is reading. + with contextlib.suppress(Exception): + self._client.stop() + with contextlib.suppress(Exception): + self._client.start() + return True + @marks_execution def run_code( # noqa: C901 self, diff --git a/docs/docs/environments/specification.mdx b/docs/docs/environments/specification.mdx index bd048b4..c28ea8a 100644 --- a/docs/docs/environments/specification.mdx +++ b/docs/docs/environments/specification.mdx @@ -27,7 +27,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `base.ref` | — | An approved base: `datalayer/python-cpu` or `datalayer/python-cuda`. | | `base.channel` | — | The release channel, resolved to a digest per variant when the version is resolved. | | `platform.architecture` | `linux/amd64` | Only `linux/amd64`. | -| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. `conda` is not buildable yet. | +| `packages.python.manager` | `uv` | `uv` or `pip`; both are resolved with uv. A `conda` **`packages`** list is not built yet — bring a conda `environment.yml` as a `dependencyFile` instead (see `build.source`). | | `packages.python.dependencies` | none | PEP 508 requirements. | | `packages.python.constraints` | none | PEP 508 requirements, applied under Datalayer's protected constraints, which win. | | `packages.python.indexes` | `https://pypi.org/simple` | `https` URLs, with no credential in them. | @@ -42,7 +42,7 @@ A document is refused with `DL_ENV_SPEC_INVALID` when a field is wrong, and with | `compatibility.variants.required` | `[datalayer]` | At least one of `datalayer`, `e2b`, `daytona`, `modal`. | | `compatibility.variants.optional` | none | The same variants, none of them also required. | | `compatibility.regions` | none | Region names. | -| `build.source` | `packages` | `packages`, `dependencyFile` (a `requirements.txt`, or a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved) and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base) all resolve on the Datalayer variant; `image` from a private registry, and `dockerfile`, are refused until they do. | +| `build.source` | `packages` | `packages`, `dependencyFile` (a `requirements.txt`; a `pyproject.toml` whose own `uv.lock` is checked against Datalayer's protected pins and exported rather than re-resolved; or a conda `environment.yml`, resolved in its own `micromamba` solve into an explicit lock with the protected pins merged over its `pip:` layer) and `image` (an existing image from a bootstrap registry allowlist, replacing the approved base) all resolve on the Datalayer variant; `image` from a private registry, and `dockerfile`, are refused until they do. | ## The spec digest diff --git a/pyproject.toml b/pyproject.toml index b005720..f571e28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "jupyter-server", "jupyter-server-client", "pydantic>=2.0", + "pyyaml", "rich", "typer>=0.12.0", ] @@ -75,7 +76,7 @@ test = [ "pytest-cov>=4.0", ] lint = ["mdformat>0.7", "mdformat-gfm>=0.3.5", "ruff"] -typing = ["mypy>=0.990"] +typing = ["mypy>=0.990", "types-PyYAML"] [project.license] file = "LICENSE" diff --git a/schemas/environment-v1alpha1.json b/schemas/environment-v1alpha1.json index d23b194..2c44291 100644 --- a/schemas/environment-v1alpha1.json +++ b/schemas/environment-v1alpha1.json @@ -152,9 +152,48 @@ "title": "Compatibility", "type": "object" }, + "ContentsBuildEntry": { + "additionalProperties": false, + "description": "One immutable file the Environment bakes from an external source.\n\nUnlike ``files`` \u2014 which a user uploads, referenced by ``contentRef`` \u2014 a\n``contentsBuild`` entry names an external ``source`` URL fetched at build\ntime and verified against ``sha256`` (required: the build verifies every\nbyte it bakes). Both are baked the same way, into every provider artifact.", + "properties": { + "path": { + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "sizeBytes": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sizebytes" + }, + "source": { + "title": "Source", + "type": "string" + } + }, + "required": [ + "source", + "path", + "sha256" + ], + "title": "ContentsBuildEntry", + "type": "object" + }, "DependencyFileSpec": { "additionalProperties": false, - "description": "A `requirements.txt`, or a `pyproject.toml` with its `uv.lock` (E3-01).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one.", + "description": "A `requirements.txt`, a `pyproject.toml` with its `uv.lock`, or a conda\n`environment.yml` (E3-01, E3-02).\n\n``requirements`` resolves the way ``packages`` does \u2014 the protected\nconstraints merged in, the same solve. ``pyproject`` does not resolve at\nall: its own ``uv.lock`` is verified against the current\n``pyproject.toml`` and exported, never re-solved, because a lock the\nauthor already made is the whole point of bringing one. ``conda`` resolves\nthe ``environment.yml`` in its own ``micromamba`` solve into an explicit\nlock, with the protected constraints merged over its ``pip:`` layer.", "properties": { "content": { "default": "", @@ -170,7 +209,8 @@ "default": "requirements", "enum": [ "requirements", - "pyproject" + "pyproject", + "conda" ], "title": "Sourceformat", "type": "string" @@ -201,6 +241,13 @@ "compatibility": { "$ref": "#/$defs/Compatibility" }, + "contentsBuild": { + "items": { + "$ref": "#/$defs/ContentsBuildEntry" + }, + "title": "Contentsbuild", + "type": "array" + }, "contract": { "default": "sandbox-contract/v1", "title": "Contract", diff --git a/tests/test_client.py b/tests/test_client.py index c75c86e..34df743 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -157,6 +157,62 @@ def tool_caller(): assert client.is_alive() is True +class TestRestartingWhatIsActuallyHoldingTheState: + """A restart has to clear the interpreter, not just this client's socket. + + Appendix B check 7 is "nothing is assumed to persist across restarts". A + sandbox this process owns is cleared by its own lifecycle; one attached + to somebody else's Jupyter server is not, because the kernel outlives + the websocket. Found live on r1, 2026-09-16: `state survived the + restart`. + """ + + def test_a_sandbox_that_can_restart_its_kernel_is_asked_to(self): + class _AttachedSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.kernel_restarts = 0 + + def restart_kernel(self): + self.kernel_restarts += 1 + return True + + sandbox = _AttachedSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.kernel_restarts == 1 + # And the lifecycle was left alone: stopping it would have dropped a + # connection the restarted kernel is still reachable on. + assert sandbox.is_started is True + + def test_a_kernel_restart_that_fails_falls_back_to_the_lifecycle(self): + class _RefusingSandbox(_FakeSandbox): + def __init__(self): + super().__init__() + self.stops = 0 + + def restart_kernel(self): + return False + + def stop(self): + self.stops += 1 + super().stop() + + sandbox = _RefusingSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + + assert sandbox.stops == 1, "a refused kernel restart still restarts the sandbox" + assert sandbox.is_started is True + + def test_a_sandbox_with_no_kernel_of_its_own_restarts_as_it_always_did(self): + sandbox = _FakeSandbox() + sandbox.start() + CodeSandboxClient(sandbox).restart() + assert sandbox.is_started is True + + def test_a_sandbox_that_knows_it_is_gone_is_believed(): """`is_alive` reports what the sandbox can find out, not the local start flag, so a backend that died under us is not reported as ready.""" diff --git a/tests/test_environment_attest.py b/tests/test_environment_attest.py index 90038b7..842f3c5 100644 --- a/tests/test_environment_attest.py +++ b/tests/test_environment_attest.py @@ -80,7 +80,7 @@ def basic(identifier: str, severity: str, *, package: str = "openssl") -> dict: class FakeEcr: - """The two ECR calls the attestor makes, and nothing else.""" + """The three ECR calls the attestor makes, and nothing else.""" def __init__( self, @@ -90,6 +90,7 @@ def __init__( enhanced_findings=True, manifest: dict | None = None, pages: list[list[dict]] | None = None, + size_bytes: int | None = 2_147_483_648, ) -> None: self.statuses = list(statuses) self.findings = list(findings) @@ -99,8 +100,12 @@ def __init__( self.pages = pages self.enhanced_findings = enhanced_findings self.manifest = manifest + #: What `describe_images` answers for the digest; None makes it raise + #: `ImageNotFoundException`, the way a registry that lost it would. + self.size_bytes = size_bytes self.asked = 0 self.scanned: list[str] = [] + self.sized: list[str] = [] def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa: N803 - boto3's spelling manifest = self.manifest or { @@ -117,6 +122,14 @@ def batch_get_image(self, repositoryName, imageIds, acceptedMediaTypes): # noqa ] } + def describe_images(self, repositoryName, imageIds): # noqa: N803 - boto3's spelling + self.sized.append(imageIds[0]["imageDigest"]) + if self.size_bytes is None: + error = Exception("ImageNotFoundException") + error.response = {"Error": {"Code": "ImageNotFoundException"}} + raise error + return {"imageDetails": [{"imageSizeInBytes": self.size_bytes}]} + def describe_image_scan_findings(self, repositoryName, imageId, nextToken=None): # noqa: N803 - boto3's spelling self.asked += 1 self.scanned.append(imageId["imageDigest"]) @@ -630,6 +643,33 @@ def test_the_policy_it_was_decided_under_is_part_of_the_record(self) -> None: answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor()) assert answer["scan_summary"]["policy"] == DEFAULT_POLICY.body() + def test_the_size_is_read_from_the_registry_when_nobody_hands_one_down(self) -> None: + """Which is every real call: the builder answers a reference, not a + weight, so `environments.artifact.bytes` — section 14's artifact size + — had no point in it although artifacts had been recorded (E1-25).""" + ecr = FakeEcr(size_bytes=2_147_483_648) + answer = attest_artifact(artifact=self.an_artifact(), attestor=an_attestor(ecr=ecr)) + assert answer["size_bytes"] == 2_147_483_648 + assert ecr.sized == [DIGEST] + + def test_a_size_handed_down_is_kept_and_the_registry_is_not_asked(self) -> None: + ecr = FakeEcr() + answer = attest_artifact( + artifact=self.an_artifact(), size_bytes=116_183_040, attestor=an_attestor(ecr=ecr) + ) + assert answer["size_bytes"] == 116_183_040 + assert ecr.sized == [] + + def test_a_size_that_cannot_be_read_is_not_a_reason_to_refuse(self) -> None: + """A missing number on a dashboard, against an artifact nothing can + launch: the artifact is signed and the attestation stands.""" + said: list[str] = [] + attestor = an_attestor(ecr=FakeEcr(size_bytes=None), log=said.append) + answer = attest_artifact(artifact=self.an_artifact(), attestor=attestor) + assert answer["size_bytes"] is None + assert answer["signature_ref"] == f"{REGISTRY}/{REPOSITORY}@{DIGEST}" + assert any("size could not be read" in line for line in said) + def test_nothing_reaches_a_registry_when_nothing_could_sign() -> None: """The order that keeps a refusal cheap and honest (E1-08, E1-09). diff --git a/tests/test_environment_bases.py b/tests/test_environment_bases.py index c2a185a..0de2f97 100644 --- a/tests/test_environment_bases.py +++ b/tests/test_environment_bases.py @@ -52,10 +52,17 @@ def test_the_2026_09_channel_of_python_cuda_has_no_digest_until_it_is_pushed( def test_the_2026_09_channel_of_python_cpu_resolves_the_digest_its_release_pushed( variant: str, ) -> None: - """PLAN_ENV.md, E1-05: released 2026-09-14 (jupyter-python 0.2.2, E1-08's - scan fix), same digest for every variant.""" + """PLAN_ENV.md, E1-05: the digest the channel's last release pushed, the + same one for every variant. + + **This moves with every base release**, and `bases.py` is where it moves + first: two releases (2026-09-15's and 2026-09-16's) changed the channel + and left this assertion on 2026-09-14's digest, so it sat red rather than + catching anything. Current: released 2026-09-16, the contract layer that + starts kernels in `/home/datalayer/content` (E1-05, Appendix B check 2). + """ ref = "datalayer/python-cpu" - digest = "sha256:334adf6c2714c8919ef60beeca1db12e3531a391c9dde41932c782f81c432b36" + digest = "sha256:122d3e31f5e2507251457cbf47871c39ac1753adb1d83777ab0743fa11cd6148" assert APPROVED_BASES[ref].channels == {"2026.09": dict.fromkeys(VARIANTS, digest)} assert resolve_base(ref, "2026.09", variant) == digest @@ -114,8 +121,12 @@ def test_each_base_is_published_under_its_own_repository() -> None: def test_the_2026_09_channel_of_python_cpu_pins_apt_to_its_snapshot() -> None: - """D-9: the moment just after the channel's image upgraded its packages.""" - assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260914T150000Z" + """D-9: the moment just after the channel's image upgraded its packages. + + Moves with the channel, like the digest above, and had rotted the same + way — left on 2026-09-14's id after the channel moved to 2026-09-16's. + """ + assert channel_snapshot("datalayer/python-cpu", "2026.09") == "20260916T120000Z" assert channel_snapshot("datalayer/python-cuda", "2026.09") == "" assert channel_snapshot("datalayer/nothing", "2026.09") == "" diff --git a/tests/test_environment_builders.py b/tests/test_environment_builders.py index 4596ecc..28eb4fb 100644 --- a/tests/test_environment_builders.py +++ b/tests/test_environment_builders.py @@ -280,6 +280,58 @@ def test_a_file_is_not_baked_without_its_digest() -> None: files_step(environment(), variant="kaggle") +CONTENTS_SHA = "c" * 64 + + +def test_the_files_step_bakes_a_contents_build_manifest_from_an_external_source() -> None: + env = environment( + files=[], + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ], + ) + commands = files_step(env, variant="modal") + # The external source is fetched as it is (no contentRef to sign) and verified. + assert ( + "curl -fsSL https://data.example/iris.csv -o /opt/datalayer/contents/iris.csv" + in commands[0] + ) + assert f'echo "{CONTENTS_SHA} /opt/datalayer/contents/iris.csv" | sha256sum -c' in commands[0] + assert "environment-contents.json" in commands[-1] + + +def test_files_and_contents_build_are_baked_together() -> None: + env = environment( + contents_build=[ + { + "source": "https://data.example/iris.csv", + "path": "/opt/datalayer/contents/iris.csv", + "sha256": CONTENTS_SHA, + } + ] + ) + entries = build_entries(env, source_of=lambda entry: "https://signed.example/notes.md") + # The uploaded file first, then the external build entry — both baked, one engine. + assert [entry.destination_path for entry in entries] == [ + "/home/datalayer/content/notes.md", + "/opt/datalayer/contents/iris.csv", + ] + assert entries[0].source_uri == "https://signed.example/notes.md" + assert entries[1].source_uri == "https://data.example/iris.csv" + assert entries[1].sha256 == CONTENTS_SHA + + +def test_a_contents_build_entry_without_its_digest_will_not_parse() -> None: + with pytest.raises(Exception): + environment( + contents_build=[{"source": "https://data.example/x", "path": "/opt/x"}] + ) + + def test_the_neutral_modules_import_no_provider_sdk() -> None: """What a service may import must not drag a provider in.""" code = ( diff --git a/tests/test_environment_conformance.py b/tests/test_environment_conformance.py index a0e8c30..cf1e9d2 100644 --- a/tests/test_environment_conformance.py +++ b/tests/test_environment_conformance.py @@ -282,6 +282,36 @@ def test_egress_and_gpu_are_judged_against_what_was_asked() -> None: assert "CUDA is 12.2, not 12.4" in by_id(result, 11).detail +def test_check_eleven_gates_a_gpu_version_that_cannot_see_its_gpu() -> None: + """E2-17: the GPU check is the one extended check that gates, and only + for a version that asked for an accelerator — a GPU version whose GPU is + not visible is not the version its spec describes.""" + sandbox = ScriptedSandbox({11: {"returncode": 0, "gpus": [], "cuda": None}}) + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating + assert not gpu.passed and not result.passed + assert "no GPU is visible" in gpu.detail + + +def test_check_eleven_gates_when_a_gpu_version_sees_its_gpu() -> None: + """The same version, its GPU and CUDA as the spec asked: the gating check + passes, so the extended tier passes.""" + sandbox = ScriptedSandbox() + result = run_extended_tier(sandbox, accelerator_requested=True, cuda_version="12.4") + gpu = by_id(result, 11) + assert gpu.gating and gpu.passed and result.passed + + +def test_a_cpu_version_never_gates_on_the_gpu_check() -> None: + """No accelerator asked for: check 11 is the trivial recorded pass, and + the extended tier still gates nothing.""" + result = run_extended_tier(ScriptedSandbox()) + gpu = by_id(result, 11) + assert not gpu.gating and gpu.passed + assert not any(item.gating for item in result.checks) + + def test_the_probes_run_for_real_in_a_local_sandbox( tmp_path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/test_environment_constraints.py b/tests/test_environment_constraints.py index 06dcef9..0aa1f0f 100644 --- a/tests/test_environment_constraints.py +++ b/tests/test_environment_constraints.py @@ -20,6 +20,7 @@ "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } diff --git a/tests/test_environment_contract.py b/tests/test_environment_contract.py index b669e0a..41ad1ef 100644 --- a/tests/test_environment_contract.py +++ b/tests/test_environment_contract.py @@ -16,11 +16,14 @@ from code_sandboxes.environments.contract import ( SANDBOX_CONTRACT_V1, SUPPORTED_CONTRACTS, + BuildContextEntry, + check_build_context, check_dockerfile, contract_markdown, get_contract, main, parse_dockerfile, + validate_build_context, validate_dockerfile, ) from code_sandboxes.environments.doctor.datalayer_sandbox import ROW_IDS @@ -134,6 +137,54 @@ def test_the_parser_joins_continuations_and_skips_comments_inside_them() -> None assert instructions[1].arguments == "apt-get update && apt-get install -y gdal-bin" +# -- The build context (E3-03) ------------------------------------------------- + + +def test_a_plain_build_context_passes() -> None: + entries = [ + BuildContextEntry("Dockerfile", 200), + BuildContextEntry("src/app.py", 1024), + BuildContextEntry("data/model.bin", 5 * 1024 * 1024), + ] + assert validate_build_context(entries) == [] + check_build_context(entries) + + +@pytest.mark.parametrize( + ("entry", "message"), + [ + (BuildContextEntry("/etc/passwd", 10), "is an absolute path, not a context path"), + (BuildContextEntry("../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("a/../../secret", 10), "escapes the context with `..`"), + (BuildContextEntry("link", is_symlink=True), "is a symlink, which could read a host file"), + ( + BuildContextEntry("big.bin", 50 * 1024 * 1024 + 1), + "is over the 52428800-byte per-file limit", + ), + ], +) +def test_a_forbidden_context_member_is_refused( + entry: BuildContextEntry, message: str +) -> None: + findings = validate_build_context([entry]) + assert any(finding.message == message for finding in findings), findings + with pytest.raises(EnvironmentsError) as refused: + check_build_context([entry]) + assert refused.value.code is errors.SPEC_INVALID + + +def test_too_many_files_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 1) for index in range(2001)] + findings = validate_build_context(entries) + assert any("more than 2000 files" in finding.message for finding in findings) + + +def test_too_many_bytes_in_all_is_refused() -> None: + entries = [BuildContextEntry(f"f{index}", 40 * 1024 * 1024) for index in range(3)] + findings = validate_build_context(entries) + assert any("total limit" in finding.message for finding in findings) + + @pytest.mark.parametrize( ("image", "approved"), [ diff --git a/tests/test_environment_datalayer_builder.py b/tests/test_environment_datalayer_builder.py index 5b92936..c68e557 100644 --- a/tests/test_environment_datalayer_builder.py +++ b/tests/test_environment_datalayer_builder.py @@ -43,6 +43,29 @@ ) LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02): the `@EXPLICIT` marker, one conda package +#: URL, and the pip layer the solve resolved (the user's pip requirements and +#: the protected pins over them). +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) + +#: The section 4.1 example as a conda `dependencyFile` source. +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's registry credential, as the workflow mints one (D-17).""" @@ -248,6 +271,26 @@ def test_it_installs_from_the_lock_with_hashes(self) -> None: # Never the loose list: that is the whole point of resolving once. assert "geopandas==1.1.1" not in dockerfile + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): the `@EXPLICIT` lock installs with a pinned + `micromamba` copied in first, and the pip layer the solve resolved + follows, so the kernel stack (E1-04) is present the same.""" + request = a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC) + dockerfile = a_builder().dockerfile(request) + assert ( + "micromamba install --yes --name base --file /opt/datalayer/lock.txt" in dockerfile + ) + # micromamba is copied in from its pinned image before it is invoked. + bootstrap = dockerfile.index("COPY --from=mambaorg/micromamba") + micromamba = dockerfile.index("micromamba install") + assert bootstrap < micromamba + # The header's own pip layer, installed with pip after the conda layer. + pip = dockerfile.index("uv pip install --system") + assert micromamba < pip + assert "ipykernel==7.3.0" in dockerfile + # A conda source never runs the pip-lock `uv pip sync`. + assert "uv pip sync" not in dockerfile + def test_apt_installs_from_the_snapshot_the_lock_pinned_it_at(self) -> None: """A pinned version can leave the live mirror: the build installs from the snapshot the resolver pinned against (D-9).""" diff --git a/tests/test_environment_daytona_builder.py b/tests/test_environment_daytona_builder.py index 3ac05ab..5ba666c 100644 --- a/tests/test_environment_daytona_builder.py +++ b/tests/test_environment_daytona_builder.py @@ -45,6 +45,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" @@ -459,6 +478,20 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync.args[0] assert "--find-links /opt/datalayer/wheelhouse" in sync.args[0] + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" + daytona = FakeDaytonaModule() + a_builder(daytona=daytona).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + image = daytona.client.snapshot.create_calls[0].args[0].image + runs = calls_named(image, "run_commands") + bootstrap = next(i for i, call in enumerate(runs) if "micro.mamba.pm" in call.args[0]) + micromamba = next(i for i, call in enumerate(runs) if "micromamba install" in call.args[0]) + pip = next(i for i, call in enumerate(runs) if "ipykernel==7.3.0" in call.args[0]) + assert bootstrap < micromamba < pip + assert not any("uv pip sync" in call.args[0] for call in runs) + def test_user_root_brackets_the_install_steps(self) -> None: """Daytona honours the base's `USER`, unlike E2B (E0-04): no synthetic account, just `USER root` around what needs it.""" diff --git a/tests/test_environment_e2b_builder.py b/tests/test_environment_e2b_builder.py index ce93277..c043b4e 100644 --- a/tests/test_environment_e2b_builder.py +++ b/tests/test_environment_e2b_builder.py @@ -35,6 +35,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, E2-01).""" @@ -390,6 +409,32 @@ def test_no_apt_step_when_the_lock_pins_none(self) -> None: a_builder(fake).build(a_request()) assert not any(call.name == "run_cmd" and "apt-get" in call.args[0] for call in fake.calls) + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): micromamba is bootstrapped, `micromamba + install --file` reads the `@EXPLICIT` lock, and the pip layer the solve + resolved follows — never the pip-lock `uv pip sync`.""" + fake = FakeTemplate() + a_builder(fake).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + bootstrap = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micro.mamba.pm" in call.args[0] + ) + micromamba = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "micromamba install" in call.args[0] + ) + pip = next( + i + for i, call in enumerate(fake.calls) + if call.name == "run_cmd" and "ipykernel==7.3.0" in call.args[0] + ) + assert bootstrap < micromamba < pip + assert not any( + call.name == "run_cmd" and "uv pip sync" in call.args[0] for call in fake.calls + ) + def test_the_owners_credential_is_passed_to_the_sdk(self) -> None: """D-8: a build must run in the *environment owner's* team, never whichever team the worker process itself happens to be configured diff --git a/tests/test_environment_managed_builders.py b/tests/test_environment_managed_builders.py index 977e508..a473928 100644 --- a/tests/test_environment_managed_builders.py +++ b/tests/test_environment_managed_builders.py @@ -52,6 +52,30 @@ def environment(**spec: Any) -> Environment: return parse_environment(data) +#: A `dependencyFile` conda source: an `environment.yml` a managed variant +#: builds with `micromamba` (E3-02). +CONDA_ENVIRONMENT_YML = ( + "name: geo\n" + "channels: [conda-forge]\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal\n" +) + + +def a_conda_environment(**spec: Any) -> Environment: + return environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": CONDA_ENVIRONMENT_YML, + }, + }, + **spec, + ) + + def messages(report: CapabilityReport) -> str: return " | ".join(finding.message for finding in report.findings) @@ -96,9 +120,10 @@ def test_each_one_bounds_how_long_a_build_may_take(self) -> None: seconds = get_builder(variant).capabilities().max_build_seconds assert seconds and 0 < seconds <= 60 * 60, variant - def test_this_phase_builds_a_package_list_and_nothing_else(self) -> None: + def test_this_phase_builds_a_package_list_and_a_conda_file(self) -> None: for variant in MANAGED: - assert get_builder(variant).capabilities().build_sources == ("packages",) + sources = get_builder(variant).capabilities().build_sources + assert sources == ("packages", "dependencyFile"), variant # -- what each one refuses ------------------------------------------------------ @@ -208,14 +233,29 @@ def test_a_source_this_phase_does_not_build_is_refused_by_name(self) -> None: report = get_builder(variant).validate(environment(build={"source": "dockerfile"})) assert report.supported is False, variant assert "`dockerfile` is not built for" in messages(report) - assert "it builds packages" in messages(report) + assert "it builds packages, dependencyFile" in messages(report) - def test_conda_is_not_resolved_for_a_managed_variant_yet(self) -> None: + def test_a_conda_dependency_file_is_buildable_on_every_managed_variant(self) -> None: + for variant in MANAGED: + report = get_builder(variant).validate(a_conda_environment()) + assert report.supported is True, f"{variant}: {messages(report)}" + + def test_a_pyproject_dependency_file_is_not_built_for_a_managed_variant_yet(self) -> None: report = get_builder("e2b").validate( - environment(packages={"python": {"manager": "conda", "dependencies": ["numpy"]}}) + environment( + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "pyproject", + "content": "[project]\nname='x'\nversion='0'\n", + "lockContent": "# lock\n", + }, + } + ) ) assert report.supported is False - assert "`conda` is not resolved for E2B yet" in messages(report) + assert "a `pyproject` dependency file is not built for E2B yet" in messages(report) + assert "spec.build.dependencyFile.sourceFormat" in fields(report) def test_e2b_and_daytona_refuse_a_build_secret_e0_04_found_no_mechanism_for(self) -> None: """E0-04's spike found only a registry login for the private base on diff --git a/tests/test_environment_modal_builder.py b/tests/test_environment_modal_builder.py index ac8e90d..0adc96f 100644 --- a/tests/test_environment_modal_builder.py +++ b/tests/test_environment_modal_builder.py @@ -46,6 +46,25 @@ APT_LOCK = LOCK + "# datalayer-apt: gdal-bin=3.8.4+dfsg-3build2\n" LOCK_DIGEST = "sha256:" + "dd" * 32 +#: A conda explicit lock (E3-02) and its `dependencyFile` source. +CONDA_LOCK = ( + "# Resolved by Datalayer (PLAN_ENV.md D-9). Do not edit: a change makes a new version.\n" + "# python: 3.13\n" + "# platform: linux-64\n" + "# datalayer-pip: ipykernel==7.3.0\n" + "@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.8.4-py313.conda#" + "ab" * 32 + "\n" +) +CONDA_SPEC = { + "build": { + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "name: geo\nchannels: [conda-forge]\ndependencies: [python=3.13, gdal]\n", + }, + }, +} + class Credential: """The build's owner secrets, as the workflow mints them (D-8, D-17, E2-01).""" @@ -114,6 +133,14 @@ def run_commands(self, *commands: str, secrets: Any = None) -> FakeImage: self.calls.append(Call("run_commands", commands, kwargs)) return self + def micromamba_install(self, *, spec_file: str) -> FakeImage: + self.calls.append(Call("micromamba_install", (), {"spec_file": spec_file})) + return self + + def pip_install(self, *packages: str, find_links: str | None = None) -> FakeImage: + self.calls.append(Call("pip_install", packages, {"find_links": find_links})) + return self + def workdir(self, path: str) -> FakeImage: self.calls.append(Call("workdir", (path,))) return self @@ -488,6 +515,22 @@ def test_uv_pip_sync_reaches_the_bases_own_shared_wheelhouse(self) -> None: assert "--require-hashes" in sync assert "--find-links /opt/datalayer/wheelhouse" in sync + def test_a_conda_lock_installs_with_micromamba_and_then_the_pip_pins(self) -> None: + """A conda source (E3-02): Modal's own `micromamba_install` reads the + `@EXPLICIT` lock, and `pip_install` layers the pip layer the solve + resolved — never the pip-lock `uv pip sync`.""" + modal = FakeModalModule() + a_builder(modal=modal).build(a_request(lock_text=CONDA_LOCK, spec=CONDA_SPEC)) + [image] = modal.Image.created + [mamba] = calls_named(image, "micromamba_install") + assert mamba.kwargs["spec_file"] == "/opt/datalayer/lock.txt" + [pip] = calls_named(image, "pip_install") + assert "ipykernel==7.3.0" in pip.args + mamba_at = image.calls.index(mamba) + pip_at = image.calls.index(pip) + assert mamba_at < pip_at + assert not run_commands_containing(image, "uv pip sync") + def test_no_user_line_is_ever_emitted(self) -> None: """Modal ignores `USER` entirely (found live): writing one would be dead code, so this builder never calls `dockerfile_commands` at all.""" diff --git a/tests/test_environment_resolve.py b/tests/test_environment_resolve.py index 43af27e..edeb79e 100644 --- a/tests/test_environment_resolve.py +++ b/tests/test_environment_resolve.py @@ -13,7 +13,6 @@ from __future__ import annotations import subprocess -from datetime import datetime, timezone import httpx import pytest @@ -141,10 +140,12 @@ def test_they_are_the_kernel_stack_at_one_version_each(self) -> None: "jupyter-client", "jupyter-server", "jupyter-server-nbmodel", + "jupyter-kernels", "datalayer", } assert all(pin.version for pin in pins.values()) assert pins["jupyter-server"].version == "2.21.0+datalayer.1" + assert pins["jupyter-kernels"].version == "1.2.23" def test_a_requirement_that_agrees_with_a_pin_is_dropped_for_it(self) -> None: # The fork satisfies `>=2.19`, which is what jupyterlab asks for, so @@ -264,7 +265,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) python_version="3.13", base_reference="environments/base/python-cpu@sha256:" + "11" * 32, merged=merge_requirements(["geopandas==1.1.1"]), - resolved_at=datetime(2026, 9, 12, 8, 30, tzinfo=timezone.utc), ) assert document["format"] == LOCK_FORMAT assert document["digest"].startswith("sha256:") @@ -273,7 +273,6 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) content = document["content"] assert f"{APT_PIN_PREFIX}gdal-bin=3.8.4+dfsg-3build2" in content assert "# datalayer-protected: ipykernel==7.3.0" in content - assert "# resolved-at: 2026-09-12T08:30:00+00:00" in content # The snapshot the pins came from, which the builder installs from. from code_sandboxes.environments.resolve import apt_pins_in, apt_snapshot_in @@ -287,15 +286,24 @@ def test_it_is_a_requirements_file_with_what_else_is_installed_in_comments(self) } def test_the_same_lock_digests_the_same_and_a_changed_one_does_not(self) -> None: + """And nothing here may pin a clock to make it true. + + The header carried a `# resolved-at:` line until 2026-09-16, and this + test passed only because it froze the moment. Resolving the same spec + twice on r1 produced two digests differing in that one line out of + 5,388, and section 5's cache key is over the lock digest — so D-12's + build cache had never hit, 12 lookups out of 12. + """ arguments = { "python_version": "3.13", "base_reference": "environments/base/python-cpu@sha256:" + "11" * 32, "merged": MergedRequirements((), (), ()), - "resolved_at": datetime(2026, 9, 12, tzinfo=timezone.utc), } first = lock_document(A_LOCK, **arguments) again = lock_document(A_LOCK, **arguments) assert first["digest"] == again["digest"] + assert first["content"] == again["content"] + assert "resolved-at" not in first["content"] moved = lock_document( ResolveOutcome(lock_text=A_LOCK.lock_text.replace("1.1.1", "1.1.2")), **arguments ) @@ -454,6 +462,54 @@ def test_conda_waits_for_its_own_solver(self) -> None: assert raised.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" assert raised.value.detail["manager"] == "conda" + def test_a_conda_dependency_file_uses_the_conda_runner_it_is_given(self) -> None: + """The main API forwards its `conda_runner` to the conda solve, so a + local or test solver reaches it the same way `runner` reaches pip.""" + from code_sandboxes.environments.resolve_conda import ( + CondaResolveOutcome, + CondaResolveRequest, + ) + + class RecordedConda: + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request, log=None): # type: ignore[no-untyped-def] + self.request = request + return self._outcome + + conda_runner = RecordedConda( + CondaResolveOutcome( + lock_text=( + "# platform: linux-64\n@EXPLICIT\n" + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + + "bb" * 32 + + "\n" + ) + ) + ) + spec = a_spec( + packages={}, + build={ + "source": "dependencyFile", + "dependencyFile": { + "sourceFormat": "conda", + "content": "channels:\n - conda-forge\ndependencies:\n - gdal=3.9\n", + }, + }, + ) + resolve_environment( + spec=spec, + variants=["datalayer"], + conda_runner=conda_runner, + bases=BASES, + ) + assert conda_runner.request is not None + assert "gdal=3.9" in conda_runner.request.environment_yml + def test_the_credentials_registry_auth_reaches_the_runner(self) -> None: class Credential: def registry_auth(self) -> dict[str, str]: @@ -595,6 +651,7 @@ def a_pyproject_spec(**dependency_file: object) -> dict[str, object]: "jupyter-client==8.9.1 \\\n --hash=sha256:" + "cc" * 32 + "\n" "jupyter-server==2.21.0+datalayer.1 \\\n --hash=sha256:" + "dd" * 32 + "\n" "jupyter-server-nbmodel==0.2.8 \\\n --hash=sha256:" + "ee" * 32 + "\n" + "jupyter-kernels==1.2.23 \\\n --hash=sha256:" + "a7" * 32 + "\n" "datalayer==1.7.4 \\\n --hash=sha256:" + "ff" * 32 + "\n" ) @@ -610,7 +667,7 @@ def test_a_current_lock_is_exported_rather_than_resolved(self) -> None: pyproject_run=uv, ) assert answer["content"] == EXPORTED - assert answer["package_count"] == 6 + assert answer["package_count"] == 7 assert answer["python_version"] == "3.13" assert uv.calls[0][:2] == ["/usr/bin/uv", "lock"] assert uv.calls[1][:2] == ["/usr/bin/uv", "export"] @@ -947,8 +1004,12 @@ def test_the_solve_is_asked_to_pin_apt_at_the_base_channels_snapshot() -> None: runner = RecordedRunner(A_LOCK) resolve_environment(spec=a_spec(), variants=["datalayer"], runner=runner) assert runner.request is not None + # What this test is about: the solve is pinned at *the channel's* snapshot, + # whatever that is. The id itself is pinned once, in + # `test_environment_bases.py`, where it moves with the channel — repeating + # it here only meant a base release left two tests red instead of one. assert runner.request.apt_snapshot == channel_snapshot("datalayer/python-cpu", "2026.09") - assert runner.request.apt_snapshot == "20260914T150000Z" + assert runner.request.apt_snapshot class TestTheBuildkitRunner: diff --git a/tests/test_environment_resolve_conda.py b/tests/test_environment_resolve_conda.py new file mode 100644 index 0000000..26b3049 --- /dev/null +++ b/tests/test_environment_resolve_conda.py @@ -0,0 +1,470 @@ +# Copyright (c) 2025-2026 Datalayer, Inc. +# +# BSD 3-Clause License + +"""Conda resolution: an environment.yml to one explicit lock (E3-02). + +Like the pip resolver's suite, micromamba's refusals below are **recorded** — +each is what ``micromamba`` actually writes for that input — so the parser is +tested against the solver's own words rather than a paraphrase of them. +""" + +from __future__ import annotations + +import subprocess + +import pytest + +from code_sandboxes.environments.bases import ApprovedBase +from code_sandboxes.environments.errors import EnvironmentsError +from code_sandboxes.environments.resolve import protected_pins +from code_sandboxes.environments.resolve_conda import ( + CONDA_LOCK_FORMAT, + BuildkitCondaResolveRunner, + CondaResolveOutcome, + CondaResolveRequest, + MicromambaResolveRunner, + conda_lock_document, + explicit_lock_packages, + merge_conda_pip, + parse_conda_environment, + parse_conda_failure, + pip_requirements_from_env_yaml, + rendered_environment, + resolve_conda_environment, +) +from code_sandboxes.environments.spec import validate_environment + +# -- What micromamba wrote ---------------------------------------------------- + +#: An explicit lock, as ``micromamba env export --explicit`` writes one. +EXPLICIT_LOCK = """\ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: linux-64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#{} +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#{} +""".format("aa" * 32, "bb" * 32) + +#: A package no channel serves. +CONDA_MISSING = """\ +critical libmamba Could not solve for environment specs +The following package could not be found: + - nothing provides no-such-conda-pkg-xyzzy needed by requested +""" + +#: An unsatisfiable set. +CONDA_CONFLICT = """\ +critical libmamba Could not solve for environment specs +The following packages are incompatible +encountered problems while solving: + - package gdal-3.9.2 requires libgdal 3.9.*, but none of the providers can be installed +""" + +#: Not a resolution failure at all — the channel is unreachable. +CONDA_UNREACHABLE = """\ +critical libmamba Download error (6) Could not resolve host: conda.anaconda.org +""" + +BASES = { + "datalayer/python-cpu": ApprovedBase( + ref="datalayer/python-cpu", + python_versions=("3.13",), + channels={ + "2026.09": {"datalayer": "sha256:" + "11" * 32, "modal": "sha256:" + "22" * 32}, + "2026.10": {}, + }, + ) +} + +A_YAML = """\ +name: geospatial +channels: + - conda-forge +dependencies: + - gdal=3.9 + - pip: + - shapely==2.0.6 +""" + + +def a_conda_spec(content: str = A_YAML, **spec: object) -> dict[str, object]: + """A section 4.1 example whose build source is a conda environment.yml.""" + return { + "apiVersion": "environments.datalayer.io/v1alpha1", + "kind": "Environment", + "metadata": {"name": "geospatial", "title": "Geospatial"}, + "spec": { + "language": {"name": "python", "version": "3.13"}, + "base": {"ref": "datalayer/python-cpu", "channel": "2026.09"}, + "build": { + "source": "dependencyFile", + "dependencyFile": {"sourceFormat": "conda", "content": content}, + }, + "resources": {"sizeClass": "medium"}, + "compatibility": {"variants": {"required": ["datalayer"], "optional": ["modal"]}}, + **spec, + }, + } + + +class RecordedRunner: + """A conda solve that answers what it was given, remembering the request.""" + + name = "recorded-conda" + + def __init__(self, outcome: CondaResolveOutcome | Exception) -> None: + self._outcome = outcome + self.request: CondaResolveRequest | None = None + + def solve(self, request: CondaResolveRequest, log=None) -> CondaResolveOutcome: + self.request = request + if log is not None: + log("solving conda") + if isinstance(self._outcome, Exception): + raise self._outcome + return self._outcome + + +# -- Reading the environment.yml --------------------------------------------- + + +class TestParsingTheEnvironmentYml: + def test_it_separates_the_conda_and_pip_layers(self) -> None: + env = parse_conda_environment(A_YAML) + assert env.channels == ("conda-forge",) + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == ("shapely==2.0.6",) + + def test_no_pip_section_is_an_empty_pip_layer(self) -> None: + env = parse_conda_environment("dependencies:\n - gdal=3.9\n") + assert env.conda_dependencies == ("gdal=3.9",) + assert env.pip_dependencies == () + + def test_malformed_yaml_is_refused_with_its_field(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("dependencies: [\n") + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert caught.value.detail["field"] == "spec.build.dependencyFile.content" + + def test_a_document_that_is_not_a_mapping_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("- just\n- a\n- list\n") + + def test_missing_dependencies_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment("name: env\nchannels: [conda-forge]\n") + assert "dependencies" in caught.value.message + + def test_two_pip_sections_are_refused(self) -> None: + text = "dependencies:\n - pip:\n - a\n - pip:\n - b\n" + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert "more than one" in caught.value.message + + def test_a_nested_list_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - [nested]\n") + + def test_a_non_string_pip_entry_is_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("dependencies:\n - pip:\n - 3\n") + + def test_channels_that_are_not_a_list_are_refused(self) -> None: + with pytest.raises(EnvironmentsError): + parse_conda_environment("channels: conda-forge\ndependencies:\n - gdal\n") + + def test_a_channel_url_carrying_a_credential_is_refused(self) -> None: + # The same refusal a credential-bearing pip index gets: the token + # belongs in the build's secrets, never verbatim in the spec. + text = ( + "channels:\n" + " - https://user:tok@conda.example.com/private\n" + "dependencies:\n - gdal\n" + ) + with pytest.raises(EnvironmentsError) as caught: + parse_conda_environment(text) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + assert "credential" in caught.value.message + assert caught.value.detail["field"].endswith("channels[0]") + + def test_a_plain_channel_url_without_a_credential_is_kept(self) -> None: + env = parse_conda_environment( + "channels:\n - https://conda.anaconda.org/conda-forge\n" "dependencies:\n - gdal\n" + ) + assert env.channels == ("https://conda.anaconda.org/conda-forge",) + + +class TestReadingThePipExport: + def test_it_reads_the_pip_section_in_order(self) -> None: + export = ( + "name: solved\n" + "channels:\n - conda-forge\n" + "dependencies:\n" + " - python=3.13\n" + " - gdal=3.9.2\n" + " - pip:\n" + " - shapely==2.0.6\n" + " - ipykernel==7.3.0\n" + ) + assert pip_requirements_from_env_yaml(export) == ( + "shapely==2.0.6", + "ipykernel==7.3.0", + ) + + def test_an_export_without_a_pip_section_is_an_empty_layer(self) -> None: + export = "dependencies:\n - python=3.13\n - gdal=3.9.2\n" + assert pip_requirements_from_env_yaml(export) == () + + def test_a_malformed_export_is_an_empty_layer_never_a_raise(self) -> None: + assert pip_requirements_from_env_yaml("dependencies: [\n") == () + assert pip_requirements_from_env_yaml("- just\n- a\n- list\n") == () + + +# -- Datalayer's pins over the pip layer ------------------------------------- + + +class TestMergingThePipLayer: + def test_the_protected_pins_are_forced_into_the_pip_layer(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + names = {req.split("==")[0] for req in merged.requirements} + assert {"shapely", "ipykernel", "jupyter-server"} <= names + + def test_a_pip_requirement_contradicting_a_pin_is_refused(self) -> None: + env = parse_conda_environment("dependencies:\n - pip:\n - ipykernel==6.0.0\n") + with pytest.raises(EnvironmentsError) as caught: + merge_conda_pip(env) + assert caught.value.code.code == "DL_ENV_PROTECTED_PACKAGE" + + def test_the_interpreter_is_pinned_and_never_doubled(self) -> None: + env = parse_conda_environment("dependencies:\n - python=3.11\n - gdal=3.9\n") + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert rendered.count("python=3.13") == 1 + assert "python=3.11" not in rendered + + def test_the_rendered_pip_layer_carries_the_pins(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + rendered = rendered_environment(env, merged, python_version="3.13") + assert "ipykernel==7.3.0" in rendered + assert "shapely==2.0.6" in rendered + + +# -- Reading micromamba's refusals ------------------------------------------- + + +class TestReadingRefusals: + def test_a_missing_package_is_package_not_found(self) -> None: + error = parse_conda_failure(CONDA_MISSING) + assert error.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + assert "no-such-conda-pkg-xyzzy" in error.message + + def test_an_unsatisfiable_set_is_resolve_conflict(self) -> None: + error = parse_conda_failure(CONDA_CONFLICT) + assert error.code.code == "DL_ENV_RESOLVE_CONFLICT" + + def test_an_unreachable_channel_is_a_provider_error(self) -> None: + error = parse_conda_failure(CONDA_UNREACHABLE) + assert error.code.code == "DL_ENV_PROVIDER_ERROR" + assert error.code.retryable is True + + +# -- The lock ----------------------------------------------------------------- + + +class TestTheLock: + def test_it_counts_only_the_package_urls(self) -> None: + assert explicit_lock_packages(EXPLICIT_LOCK) == [ + "https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0.conda#" + "aa" * 32, + "https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.2.conda#" + "bb" * 32, + ] + + def test_the_document_records_the_pins_and_is_deterministic(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + document = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + # The header carries the complete pip layer: the user's own pip + # requirement and the protected pin the resolver forced over it. + assert "# datalayer-pip: shapely==2.0.6" in document["content"] + assert "# datalayer-pip: ipykernel==7.3.0" in document["content"] + assert "@EXPLICIT" in document["content"] + again = conda_lock_document( + CondaResolveOutcome(lock_text=EXPLICIT_LOCK), + python_version="3.13", + base_reference="registry/base@sha256:" + "11" * 32, + merged=merged, + ) + # Deterministic without anybody pinning a clock: the header carried a + # `# resolved-at:` line until 2026-09-16, which made two resolves of + # one spec two different digests and kept D-12's cache from ever + # hitting. + assert document["digest"] == again["digest"] + assert "resolved-at" not in document["content"] + + def test_an_export_without_the_marker_is_a_provider_error(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + with pytest.raises(EnvironmentsError) as caught: + conda_lock_document( + CondaResolveOutcome(lock_text="just some lines\nno marker\n"), + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + merged=merged, + ) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + + +# -- The runners refuse honestly when their tool is absent ------------------- + + +class TestTheRunners: + def test_the_local_runner_refuses_without_micromamba(self) -> None: + runner = MicromambaResolveRunner(micromamba="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_an_export_that_times_out_is_a_provider_error( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # The solve finishes, but the `env export` that reads the lock back + # exceeds the deadline: its timeout is classified the same as the + # solve's, never left as a raw subprocess.TimeoutExpired. + runner = MicromambaResolveRunner(micromamba="/usr/local/bin/micromamba", timeout=5.0) + calls = {"n": 0} + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls["n"] += 1 + if calls["n"] == 1: + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + raise subprocess.TimeoutExpired(argv, 5.0) + + monkeypatch.setattr("code_sandboxes.environments.resolve_conda.subprocess.run", fake_run) + with pytest.raises(EnvironmentsError) as caught: + runner.solve(CondaResolveRequest(environment_yml=A_YAML, python_version="3.13")) + assert caught.value.code.code == "DL_ENV_PROVIDER_ERROR" + + def test_the_buildkit_runner_refuses_without_buildctl(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert caught.value.code.code == "DL_ENV_CAPABILITY_UNSUPPORTED" + + def test_the_buildkit_runner_refuses_an_unpinned_base(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + with pytest.raises(EnvironmentsError) as caught: + runner.solve( + CondaResolveRequest( + environment_yml=A_YAML, python_version="3.13", base_reference="base:latest" + ) + ) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_the_buildkit_dockerfile_brings_the_wheelhouse_and_solves(self) -> None: + runner = BuildkitCondaResolveRunner(buildctl="/usr/bin/buildctl") + dockerfile = runner.dockerfile( + CondaResolveRequest( + environment_yml=A_YAML, + python_version="3.13", + base_reference="base@sha256:" + "11" * 32, + ) + ) + assert "micromamba create" in dockerfile + assert "PIP_FIND_LINKS" in dockerfile + assert "micromamba env export --explicit" in dockerfile + # The pinned micromamba is copied in (the base bakes uv, not it), and + # the pip layer is exported alongside the explicit lock. + assert "COPY --from=mambaorg/micromamba" in dockerfile + assert "env export --prefix /solve/prefix > /solve/pip-env.yml" in dockerfile + assert "COPY --from=solve /solve/pip-env.yml /pip-env.yml" in dockerfile + + +# -- The whole resolve, through the recorded runner -------------------------- + + +class TestResolvingACondaVersion: + def test_it_answers_the_lock_and_the_bases(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + document = resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert document["format"] == CONDA_LOCK_FORMAT + assert document["package_count"] == 2 + assert document["resolved_bases"] == {"datalayer": "base@sha256:" + "11" * 32} + + def test_it_sends_the_rendered_environment_with_the_pins(self) -> None: + runner = RecordedRunner(CondaResolveOutcome(lock_text=EXPLICIT_LOCK)) + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert runner.request is not None + assert "python=3.13" in runner.request.environment_yml + assert "ipykernel==7.3.0" in runner.request.environment_yml + + def test_a_missing_package_propagates_as_package_not_found(self) -> None: + runner = RecordedRunner(parse_conda_failure(CONDA_MISSING)) + with pytest.raises(EnvironmentsError) as caught: + resolve_conda_environment( + environment_yml=A_YAML, + python_version="3.13", + resolved_bases={"datalayer": "base@sha256:" + "11" * 32}, + runner=runner, + ) + assert caught.value.code.code == "DL_ENV_PACKAGE_NOT_FOUND" + + +# -- The spec validates a conda dependency file ------------------------------ + + +class TestValidatingTheSpec: + def test_a_conda_environment_file_validates(self) -> None: + # Does not raise: a well-formed conda source is authorable. + validate_environment(a_conda_spec(), bases=BASES) + + def test_an_empty_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content=" \n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_malformed_environment_file_is_refused(self) -> None: + with pytest.raises(EnvironmentsError) as caught: + validate_environment(a_conda_spec(content="dependencies: [\n"), bases=BASES) + assert caught.value.code.code == "DL_ENV_SPEC_INVALID" + + def test_a_lock_content_is_refused_for_conda(self) -> None: + spec = a_conda_spec() + spec["spec"]["build"]["dependencyFile"]["lockContent"] = "irrelevant" # type: ignore[index] + with pytest.raises(EnvironmentsError) as caught: + validate_environment(spec, bases=BASES) + assert "pyproject" in caught.value.message + + +class TestTheProtectedPinsAreTheKernelStack: + def test_merge_uses_the_same_pins_the_pip_resolver_does(self) -> None: + env = parse_conda_environment(A_YAML) + merged = merge_conda_pip(env) + pin_names = {pin.name for pin in protected_pins()} + forced = {req.split("==")[0].replace("_", "-") for req in merged.requirements} + assert pin_names <= forced diff --git a/tests/test_environment_spec.py b/tests/test_environment_spec.py index 81d03d7..88a8d9f 100644 --- a/tests/test_environment_spec.py +++ b/tests/test_environment_spec.py @@ -196,7 +196,6 @@ def _codes(data: dict[str, Any]) -> dict[str, str]: ("spec.contract", "sandbox-contract/v9", "spec.contract", UNSUPPORTED), ("spec.base.ref", "python", "spec.base.ref", INVALID), ("spec.language.version", "3.9", "spec.language.version", INVALID), - ("spec.build.source", "dockerfile", "spec.build.source", UNSUPPORTED), ("spec.packages.python.manager", "conda", "spec.packages.python.manager", UNSUPPORTED), # E3-05: a secret no postInstall command names would be mounted nowhere. ( @@ -307,8 +306,8 @@ def test_all_baked_files_together_are_capped() -> None: def test_an_invalid_field_outranks_something_unsupported() -> None: - data = mutated("spec.build.source", "dockerfile") - assert _codes(data) == {"spec.build.source": UNSUPPORTED} + data = mutated("spec.packages.python.manager", "conda") + assert _codes(data) == {"spec.packages.python.manager": UNSUPPORTED} with pytest.raises(EnvironmentsError) as unsupported: validate_environment(data) assert unsupported.value.code is errors.CAPABILITY_UNSUPPORTED @@ -319,10 +318,18 @@ def test_an_invalid_field_outranks_something_unsupported() -> None: assert invalid.value.code is errors.SPEC_INVALID assert {finding["field"] for finding in invalid.value.detail["findings"]} == { "metadata.name", - "spec.build.source", + "spec.packages.python.manager", } +def test_a_dockerfile_source_is_accepted_and_keeps_its_own_base() -> None: + # E3-03: the base is the `FROM` its uploaded Dockerfile names, so + # `spec.base` is not checked against the approved table (as for `image`). + data = mutated("spec.build.source", "dockerfile") + data["spec"]["base"]["ref"] = "python" + assert _codes(data) == {} + + # -- Dependency files (E3-01) -------------------------------------------------- @@ -586,3 +593,59 @@ def test_a_private_build_with_a_secret_is_untouched(self) -> None: accepting the same spec `publication_findings` refuses to publish.""" environment = parse_environment(document()) assert spec_findings(environment) == [] + + def test_a_private_index_blocks_publication(self) -> None: + """D-12: a published version resolves only from public indexes, since a + private one is reached with a credential the public does not hold.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://pypi.mycorp.internal/simple", + ] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.packages.python.indexes" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "pypi.mycorp.internal" in findings[0].message + + def test_only_public_indexes_are_publishable(self) -> None: + """The public index and its wheel host are both accepted; nothing else.""" + data = document() + del data["spec"]["buildSecrets"] + data["spec"]["packages"]["python"]["indexes"] = [ + "https://pypi.org/simple", + "https://files.pythonhosted.org/", + ] + environment = parse_environment(data) + assert publication_findings(environment) == [] + + def test_a_private_conda_channel_blocks_publication(self) -> None: + """D-12 the same for a conda source: a channel reached with a + credential the public does not hold can never be published.""" + data = a_dependency_file_document( + sourceFormat="conda", + content=( + "channels:\n" + " - conda-forge\n" + " - https://conda.mycorp.internal/private\n" + "dependencies:\n - gdal\n" + ), + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + findings = publication_findings(environment) + assert len(findings) == 1 + assert findings[0].field == "spec.build.dependencyFile.content.channels" + assert findings[0].code is errors.PUBLICATION_BLOCKED + assert "conda.mycorp.internal" in findings[0].message + + def test_public_conda_channels_are_publishable(self) -> None: + data = a_dependency_file_document( + sourceFormat="conda", + content="channels:\n - conda-forge\n - bioconda\ndependencies:\n - gdal\n", + ) + del data["spec"]["buildSecrets"] + environment = parse_environment(data) + assert publication_findings(environment) == [] diff --git a/tests/test_jupyter_server.py b/tests/test_jupyter_server.py index 9c0be8c..4de3b51 100644 --- a/tests/test_jupyter_server.py +++ b/tests/test_jupyter_server.py @@ -526,3 +526,77 @@ def test_stop_forgets_the_temporary_workdir_it_removed(tmp_path: Path, monkeypat second = Path(sandbox._resolve_workdir()) assert second.is_dir() assert second != first + + +class TestRestartingTheKernelRatherThanTheConnection: + """Appendix B check 7, against a server this process does not own. + + `stop()` then `start()` is this client's lifecycle: when the Jupyter + server belongs to somebody else — a Datalayer runtime pod, which is every + attached sandbox — it drops the websocket and leaves the kernel process + running, so the next execution lands in the same interpreter with every + global still set. Found live on r1, 2026-09-16: the smoke test read + `state survived the restart`. + """ + + def test_the_kernel_is_restarted_through_the_server_s_own_api(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + asked: dict = {} + + class _Response: + ok = True + status_code = 200 + + def _post(url, params=None, headers=None, timeout=None): + asked["url"] = url + asked["params"] = params + return _Response() + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _post) + try: + assert sandbox.restart_kernel() is True + assert asked["url"].endswith("/api/kernels/kernel-1/restart") + finally: + sandbox.stop() + + def test_a_server_that_refuses_the_restart_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + class _Response: + ok = False + status_code = 503 + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", + lambda *args, **kwargs: _Response(), + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_server_that_cannot_be_reached_is_reported_not_raised(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id="kernel-1") + + def _explode(*args, **kwargs): + raise OSError("no route to host") + + monkeypatch.setattr("code_sandboxes.jupyter_server_sandbox.requests.post", _explode) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop() + + def test_a_sandbox_with_no_kernel_id_asks_nothing(self, monkeypatch): + sandbox = _started_sandbox(monkeypatch, kernel_id=None) + + def _should_not_be_called(*args, **kwargs): + raise AssertionError("the server must not be asked without a kernel id") + + monkeypatch.setattr( + "code_sandboxes.jupyter_server_sandbox.requests.post", _should_not_be_called + ) + try: + assert sandbox.restart_kernel() is False + finally: + sandbox.stop()