diff --git a/Cargo.lock b/Cargo.lock index 528309a970..dcd91667f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3878,6 +3878,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "tar", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 55d7e7a29d..2e1420e99d 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -117,7 +117,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. | +| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -170,7 +170,7 @@ The supervisor must be available inside each sandbox workload: | Runtime | Delivery model | |---|---| | Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | -| Podman | Read-only OCI image volume containing the supervisor binary. | +| Podman | Read-only OCI image volume by default; host-cached bind mount when `userns` is configured. | | Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | | VM | Embedded in the guest rootfs bundle. | | Extension | Defined by the out-of-tree driver. | diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 35a3732cf9..ffbddf5ecd 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -27,6 +27,8 @@ ipnet = "2" base64 = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true } reqwest = { workspace = true, features = ["blocking", "rustls-tls-webpki-roots"], optional = true } +tar = "0.4" +tempfile = "3" [target.'cfg(unix)'.dependencies] nix = { workspace = true } diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index a5bcc55ad3..905d914a1e 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -3,7 +3,7 @@ //! Utility helpers shared across compute-driver crates. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::proto::compute::v1::{DriverSandbox, GetCapabilitiesResponse}; @@ -458,6 +458,143 @@ pub fn supervisor_image_should_refresh(image: &str) -> bool { matches!(supervisor_image_tag(image), Some("dev" | "latest")) } +// --------------------------------------------------------------------------- +// Supervisor binary extraction helpers (shared by Docker and Podman drivers) +// --------------------------------------------------------------------------- + +/// Extract the payload of the first regular-file entry in a tar archive. +/// +/// Container archive endpoints return a single-file tar when `path` points to +/// a file, so only the first entry is consumed. Returns an error when the +/// archive is empty, the first entry is not a regular file, or the payload is +/// empty. +pub fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { + let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); + let mut entries = archive + .entries() + .map_err(|err| format!("open tar archive: {err}"))?; + let mut entry = entries + .next() + .ok_or_else(|| "tar archive was empty".to_string())? + .map_err(|err| format!("read tar entry: {err}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() { + return Err(format!( + "expected a regular file in tar archive, got type {kind:?}" + )); + } + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes) + .map_err(|err| format!("read tar entry payload: {err}"))?; + if bytes.is_empty() { + return Err("tar entry payload was empty".to_string()); + } + Ok(bytes) +} + +/// Atomically write `bytes` to `final_path` via a sibling temp file. +/// +/// Creates parent directories as needed. The temp file is synced, `chmod 755` +/// (on Unix), and renamed into place so concurrent readers never observe a +/// partial write. Returns a human-readable error string on failure. +pub fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> Result<(), String> { + let dir = final_path + .parent() + .ok_or_else(|| format!("cache path '{}' has no parent", final_path.display()))?; + std::fs::create_dir_all(dir) + .map_err(|err| format!("failed to create cache dir '{}': {err}", dir.display()))?; + + let mut temp = tempfile::Builder::new() + .prefix(".openshell-sandbox-") + .tempfile_in(dir) + .map_err(|err| format!("failed to create temp file in '{}': {err}", dir.display()))?; + std::io::Write::write_all(&mut temp, bytes) + .map_err(|err| format!("failed to write supervisor binary: {err}"))?; + temp.as_file() + .sync_all() + .map_err(|err| format!("failed to sync supervisor binary: {err}"))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)) + .map_err(|err| format!("failed to chmod supervisor binary: {err}"))?; + } + + temp.persist(final_path).map_err(|err| { + format!( + "failed to persist supervisor binary to '{}': {}", + final_path.display(), + err.error, + ) + })?; + Ok(()) +} + +/// Return the host-side cache path for an extracted supervisor binary. +/// +/// The path is `$XDG_DATA_HOME/openshell///openshell-sandbox`. +/// `driver_subdir` distinguishes caches across drivers (e.g. `"docker-supervisor"`, +/// `"podman-supervisor"`). +pub fn supervisor_cache_path(driver_subdir: &str, digest: &str) -> Result { + let base = crate::paths::xdg_data_dir() + .map_err(|err| format!("failed to resolve XDG data dir: {err}"))?; + Ok(supervisor_cache_path_with_base( + &base, + driver_subdir, + digest, + )) +} + +/// [`supervisor_cache_path`] with an explicit base directory (for testing). +pub fn supervisor_cache_path_with_base(base: &Path, driver_subdir: &str, digest: &str) -> PathBuf { + let sanitized = digest.replace(':', "-"); + base.join("openshell") + .join(driver_subdir) + .join(sanitized) + .join("openshell-sandbox") +} + +/// Generate a unique container name for supervisor binary extraction. +/// +/// Uses the process ID and an atomic counter to avoid collisions across +/// concurrent gateway starts. +pub fn temp_extract_container_name() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let pid = std::process::id(); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + format!("openshell-supervisor-extract-{pid}-{seq}") +} + +/// Validate that the file at `path` starts with the ELF magic bytes (`\x7fELF`). +/// +/// Returns a human-readable error when the file cannot be read or is not a +/// Linux ELF binary. +pub fn validate_linux_elf_binary(path: &Path) -> Result<(), String> { + use std::io::Read; + let mut file = std::fs::File::open(path).map_err(|err| { + format!( + "failed to open supervisor binary '{}': {err}", + path.display() + ) + })?; + let mut magic = [0u8; 4]; + file.read_exact(&mut magic).map_err(|err| { + format!( + "failed to read supervisor binary '{}': {err}", + path.display() + ) + })?; + if magic != [0x7f, b'E', b'L', b'F'] { + return Err(format!( + "supervisor binary '{}' is not a Linux ELF executable", + path.display(), + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7e1bc069cb..d09edc3968 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -23,13 +23,13 @@ serde = { workspace = true } serde_json = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } -tar = "0.4" -tempfile = "3" url = { workspace = true } [dev-dependencies] prost-types = { workspace = true } +tar = "0.4" temp-env = "0.3" +tempfile = "3" [lints] workspace = true diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 502f4ae8fe..f060660ec6 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -26,7 +26,8 @@ use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - supervisor_image_should_refresh, + extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -53,7 +54,6 @@ use openshell_core::proto_struct::{ }; use openshell_core::{Config, Error, Result as CoreResult}; use std::collections::{HashMap, HashSet}; -use std::io::Read; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -3073,7 +3073,7 @@ fn resolve_supervisor_bin_source( // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. if let Some(path) = docker_config.supervisor_bin.clone() { let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path)?; + validate_linux_elf_binary(&path).map_err(Error::config)?; return Ok(SupervisorBinSource::Binary(path)); } @@ -3213,25 +3213,14 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core )) })?; - let cache_path = supervisor_cache_path(&digest)?; + let cache_path = + openshell_core::driver_utils::supervisor_cache_path("docker-supervisor", &digest) + .map_err(Error::config)?; if cache_path.is_file() { - validate_linux_elf_binary(&cache_path)?; + validate_linux_elf_binary(&cache_path).map_err(Error::config)?; return Ok(cache_path); } - let cache_dir = cache_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - cache_path.display(), - )) - })?; - std::fs::create_dir_all(cache_dir).map_err(|err| { - Error::config(format!( - "failed to create docker supervisor cache dir '{}': {err}", - cache_dir.display(), - )) - })?; - info!( image = image, digest = digest, @@ -3240,8 +3229,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core ); let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes)?; - validate_linux_elf_binary(&cache_path)?; + write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; + validate_linux_elf_binary(&cache_path).map_err(Error::config)?; Ok(cache_path) } @@ -3334,98 +3323,6 @@ async fn download_binary_from_container( }) } -/// Extract the payload of the first regular-file entry in a tar archive. -/// Docker's `/containers//archive` endpoint returns a single-file tar -/// when `path` points to a file, so we only need the first entry. -fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { - let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let mut entries = archive - .entries() - .map_err(|err| format!("open tar archive: {err}"))?; - let mut entry = entries - .next() - .ok_or_else(|| "tar archive was empty".to_string())? - .map_err(|err| format!("read tar entry: {err}"))?; - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|err| format!("read tar entry payload: {err}"))?; - Ok(bytes) -} - -fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { - let dir = final_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - final_path.display(), - )) - })?; - let mut temp = tempfile::Builder::new() - .prefix(".openshell-sandbox-") - .tempfile_in(dir) - .map_err(|err| { - Error::config(format!( - "failed to create temp file for supervisor binary in '{}': {err}", - dir.display(), - )) - })?; - std::io::Write::write_all(&mut temp, bytes).map_err(|err| { - Error::config(format!( - "failed to write supervisor binary to temp file: {err}", - )) - })?; - temp.as_file().sync_all().map_err(|err| { - Error::config(format!("failed to sync supervisor binary temp file: {err}")) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( - |err| { - Error::config(format!( - "failed to chmod supervisor binary temp file: {err}", - )) - }, - )?; - } - - temp.persist(final_path).map_err(|err| { - Error::config(format!( - "failed to rename supervisor binary into '{}': {}", - final_path.display(), - err.error, - )) - })?; - Ok(()) -} - -/// Cache path for an extracted supervisor binary, keyed by the image's -/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed -/// directory keeps stale extractions from earlier releases isolated so they -/// can be GC'd without affecting the active binary. -fn supervisor_cache_path(digest: &str) -> CoreResult { - let base = openshell_core::paths::xdg_data_dir() - .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; - Ok(supervisor_cache_path_with_base(&base, digest)) -} - -fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { - let sanitized = digest.replace(':', "-"); - base.join("openshell") - .join("docker-supervisor") - .join(sanitized) - .join("openshell-sandbox") -} - -fn temp_extract_container_name() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let pid = std::process::id(); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - format!("openshell-supervisor-extract-{pid}-{seq}") -} - fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { if !path.is_file() { return Err(Error::config(format!( @@ -3441,29 +3338,6 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult CoreResult<()> { - let mut file = std::fs::File::open(path).map_err(|err| { - Error::config(format!( - "failed to open docker supervisor binary '{}': {err}", - path.display() - )) - })?; - let mut magic = [0_u8; 4]; - file.read_exact(&mut magic).map_err(|err| { - Error::config(format!( - "failed to read docker supervisor binary '{}': {err}", - path.display() - )) - })?; - if magic != [0x7f, b'E', b'L', b'F'] { - return Err(Error::config(format!( - "docker supervisor binary '{}' must be a Linux ELF executable", - path.display() - ))); - } - Ok(()) -} - fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { docker_config.guest_tls_ca.is_some() && docker_config.guest_tls_cert.is_some() diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index fdf850dc6a..850165611c 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -5,7 +5,7 @@ use super::*; use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_NAMESPACE, supervisor_cache_path_with_base, }; use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, @@ -2174,8 +2174,11 @@ fn docker_supervisor_image_refreshes_mutable_tags_only() { #[test] fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { let base = PathBuf::from("/var/cache/share"); - let path = - supervisor_cache_path_with_base(&base, "sha256:abc123deadbeef0123456789cafe0123456789fe"); + let path = supervisor_cache_path_with_base( + &base, + "docker-supervisor", + "sha256:abc123deadbeef0123456789cafe0123456789fe", + ); assert_eq!( path, @@ -2188,8 +2191,8 @@ fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { #[test] fn supervisor_cache_path_isolates_different_digests() { let base = PathBuf::from("/data"); - let left = supervisor_cache_path_with_base(&base, "sha256:aaaaaaaa"); - let right = supervisor_cache_path_with_base(&base, "sha256:bbbbbbbb"); + let left = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:aaaaaaaa"); + let right = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:bbbbbbbb"); assert_ne!( left.parent().unwrap(), right.parent().unwrap(), diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 965a295d19..788ba5506f 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -360,6 +360,7 @@ Podman resources after out-of-band container removal or label drift. | `OPENSHELL_SANDBOX_PROXY_AUTH_FILE` | `--sandbox-proxy-auth-file` | unset | Path to a file containing the proxy credentials as `user:pass`. Staged as a root-only Podman secret so credentials never appear in config or container metadata. Requires the insecure-auth acknowledgement below. | | `OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE` | `--sandbox-proxy-auth-allow-insecure` | unset | Explicit acknowledgement (`true`) that the credential is sent as cleartext Basic auth over the plain-TCP connection to the `http://` proxy. Required when the auth file is set; rejected when it is not. | | `OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME` | `--sandbox-proxy-connect-by-hostname` | unset | Send the destination hostname in CONNECT requests instead of a validated IP. Last resort for proxies whose ACLs filter on hostnames: the proxy then resolves the name itself, so sandbox SSRF/`allowed_ips` validation no longer binds the connection. | +| `OPENSHELL_PODMAN_USERNS` | `--userns` | unset | User namespace mode for sandbox containers (e.g. `auto`). When unset, containers use the default user namespace. | Through the gateway, the same settings are the `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 9fe39cf7e2..508b604ce7 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -513,6 +513,32 @@ impl PodmanClient { } } + /// Download a file from a container as a tar archive. + /// + /// Calls `GET /libpod/containers/{name}/archive?path={path}` and returns + /// the raw tar bytes. The container does not need to be running. + pub async fn copy_from_container( + &self, + name: &str, + path: &str, + ) -> Result { + validate_name(name)?; + let encoded_path = url_encode(path); + let (status, bytes) = self + .request( + hyper::Method::GET, + &format!("/libpod/containers/{name}/archive?path={encoded_path}"), + None, + API_TIMEOUT, + ) + .await?; + if status.is_success() { + Ok(bytes) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + /// Inspect a container by name or ID. pub async fn inspect_container(&self, name: &str) -> Result { validate_name(name)?; diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 2d226397b4..c2b60d45f8 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -185,6 +185,9 @@ pub struct PodmanComputeConfig { /// pointing the gateway host at the corporate resolver so validated-IP /// CONNECT works in split-horizon networks. pub proxy_connect_by_hostname: Option, + /// User namespace mode for sandbox containers (e.g. `auto`). + /// When unset, containers use the default user namespace. + pub userns: Option, } pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; @@ -328,6 +331,37 @@ impl PodmanComputeConfig { Ok(()) } + /// Validate the optional `userns` mode against the supported allowlist. + /// + /// Supported modes: `auto` (with optional params, e.g. `auto:size=65536`), + /// `host`, `keep-id` (with optional params), `private`, and `nomap`. + /// Modes that don't accept parameters (`host`, `private`, `nomap`) are + /// rejected when a colon-separated suffix is present. + pub fn validate_userns(&self) -> Result<(), crate::client::PodmanApiError> { + let Some(mode) = self.userns.as_deref() else { + return Ok(()); + }; + let (base, has_params) = mode + .split_once(':') + .map_or((mode, false), |(b, _)| (b, true)); + match base.to_ascii_lowercase().as_str() { + "auto" | "keep-id" => Ok(()), + "host" | "private" | "nomap" => { + if has_params { + Err(crate::client::PodmanApiError::InvalidInput(format!( + "userns mode '{base}' does not accept parameters", + ))) + } else { + Ok(()) + } + } + _ => Err(crate::client::PodmanApiError::InvalidInput(format!( + "unsupported userns mode '{mode}'; \ + supported modes: auto, host, keep-id, nomap, private", + ))), + } + } + /// Validate optional host gateway override. pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); @@ -380,6 +414,7 @@ impl Default for PodmanComputeConfig { proxy_auth_file: None, proxy_auth_allow_insecure: None, proxy_connect_by_hostname: None, + userns: None, } } } @@ -412,6 +447,7 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("proxy_auth_file", &self.proxy_auth_file.is_some()) .field("proxy_auth_allow_insecure", &self.proxy_auth_allow_insecure) .field("proxy_connect_by_hostname", &self.proxy_connect_by_hostname) + .field("userns", &self.userns) .finish() } } @@ -794,4 +830,60 @@ mod tests { assert!(!msg.contains("OPENSHELL_PODMAN_TLS_CERT"), "{msg}"); assert!(!msg.contains("OPENSHELL_PODMAN_TLS_KEY"), "{msg}"); } + + #[test] + fn validate_userns_accepts_supported_modes() { + for mode in [ + "auto", + "host", + "keep-id", + "private", + "nomap", + "auto:size=65536", + "keep-id:uid=1000,gid=1000", + ] { + let cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + cfg.validate_userns() + .unwrap_or_else(|_| panic!("mode '{mode}' should be accepted")); + } + } + + #[test] + fn validate_userns_rejects_unsupported_modes() { + for mode in ["container:foo", "ns:/proc/1/ns/user", "4000:5000"] { + let cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg + .validate_userns() + .expect_err(&format!("mode '{mode}' should be rejected")); + let msg = err.to_string(); + assert!(msg.contains("unsupported userns mode"), "{msg}"); + } + } + + #[test] + fn validate_userns_rejects_params_on_non_parameterizable_modes() { + for mode in ["host:foo", "private:bar", "nomap:x=1"] { + let cfg = PodmanComputeConfig { + userns: Some(mode.to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg + .validate_userns() + .expect_err(&format!("mode '{mode}' should be rejected")); + let msg = err.to_string(); + assert!(msg.contains("does not accept parameters"), "{msg}"); + } + } + + #[test] + fn validate_userns_accepts_none() { + let cfg = PodmanComputeConfig::default(); + cfg.validate_userns().expect("None should be accepted"); + } } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 90ef0fec21..7a9244d8f3 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -229,6 +229,13 @@ struct ContainerSpec { /// Port mappings from host to container. Using `host_port=0` requests an /// ephemeral port, readable back from the inspect response. portmappings: Vec, + /// User namespace mode override (e.g. `auto`). + #[serde(skip_serializing_if = "Option::is_none")] + userns: Option, + /// UID/GID mapping options. Required for `userns = "auto"` — the Podman + /// API needs `AutoUserNs: true` alongside the namespace mode. + #[serde(skip_serializing_if = "Option::is_none")] + idmappings: Option, } /// A port mapping entry for the libpod `SpecGenerator`. @@ -328,6 +335,19 @@ struct NetNS { nsmode: String, } +#[derive(Serialize)] +struct UserNS { + nsmode: String, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, +} + +#[derive(Serialize)] +struct IDMappings { + #[serde(rename = "AutoUserNs")] + auto_user_ns: bool, +} + #[derive(Serialize)] struct NetworkAttachment {} @@ -905,9 +925,11 @@ pub fn build_container_spec_with_token_and_gpu_devices( image, image, "", + None, ) } +#[allow(clippy::too_many_arguments)] pub fn build_container_spec_for_image( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -916,6 +938,7 @@ pub fn build_container_spec_for_image( requested_image: &str, image_id: &str, oci_user: &str, + supervisor_bin_path: Option<&Path>, ) -> Result { let name = container_name(&sandbox.workspace, &sandbox.name, &sandbox.id); let vol = volume_name(&sandbox.id); @@ -957,11 +980,15 @@ pub fn build_container_spec_for_image( }]; volumes.extend(user_mounts.volumes); - let mut image_volumes = vec![ImageVolume { - source: config.supervisor_image.clone(), - destination: SUPERVISOR_MOUNT_DIR.into(), - rw: false, - }]; + let mut image_volumes = if supervisor_bin_path.is_some() { + Vec::new() + } else { + vec![ImageVolume { + source: config.supervisor_image.clone(), + destination: SUPERVISOR_MOUNT_DIR.into(), + rw: false, + }] + }; image_volumes.extend(user_mounts.image_volumes); let container_spec = ContainerSpec { @@ -1166,6 +1193,18 @@ pub fn build_container_spec_for_image( options: ro, }); } + if let Some(bin_path) = supervisor_bin_path { + let mut opts = vec!["ro".into(), "rbind".into()]; + if is_selinux_enabled() { + opts.push("z".into()); + } + m.push(Mount { + kind: "bind".into(), + source: bin_path.display().to_string(), + destination: SUPERVISOR_BINARY_PATH.into(), + options: opts, + }); + } m.extend(user_mounts.mounts); m }, @@ -1177,6 +1216,25 @@ pub fn build_container_spec_for_image( container_port: openshell_core::config::DEFAULT_SSH_PORT, protocol: "tcp".into(), }], + userns: config.userns.as_deref().map(|raw| { + let (base, params) = raw + .split_once(':') + .map_or((raw, None), |(b, p)| (b, Some(p))); + UserNS { + nsmode: base.to_string(), + value: params.map(ToString::to_string), + } + }), + idmappings: config + .userns + .as_deref() + .filter(|mode| { + mode.split(':') + .next() + .unwrap_or(mode) + .eq_ignore_ascii_case("auto") + }) + .map(|_| IDMappings { auto_user_ns: true }), }; Ok(serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail")) @@ -1379,6 +1437,7 @@ mod tests { "registry.example/app:latest", "sha256:immutable", "app:staff", + None, ) .unwrap(); @@ -2777,4 +2836,164 @@ mod tests { .count(); assert_eq!(bind_count, 0, "no bind mounts without TLS config"); } + + #[test] + fn container_spec_includes_userns_when_configured() { + let sandbox = test_sandbox("userns-id", "userns-name"); + let mut config = test_config(); + config.userns = Some("auto".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("auto")); + assert!(userns.get("value").is_none(), "bare auto should omit value"); + + let idmappings = &spec["idmappings"]; + assert_eq!( + idmappings["AutoUserNs"].as_bool(), + Some(true), + "idmappings.AutoUserNs must be true for userns=auto" + ); + } + + #[test] + fn container_spec_auto_with_params() { + let sandbox = test_sandbox("userns-auto-params-id", "userns-auto-params-name"); + let mut config = test_config(); + config.userns = Some("auto:size=65536".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("auto")); + assert_eq!(userns["value"].as_str(), Some("size=65536")); + + assert_eq!( + spec["idmappings"]["AutoUserNs"].as_bool(), + Some(true), + "idmappings.AutoUserNs must be true for auto:size=65536" + ); + } + + #[test] + fn container_spec_keep_id_with_params() { + let sandbox = test_sandbox("userns-keepid-id", "userns-keepid-name"); + let mut config = test_config(); + config.userns = Some("keep-id:uid=1000,gid=1000".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("keep-id")); + assert_eq!(userns["value"].as_str(), Some("uid=1000,gid=1000")); + + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set for keep-id" + ); + } + + #[test] + fn container_spec_nomap_mode() { + let sandbox = test_sandbox("userns-nomap-id", "userns-nomap-name"); + let mut config = test_config(); + config.userns = Some("nomap".to_string()); + let spec = build_container_spec(&sandbox, &config); + + let userns = &spec["userns"]; + assert_eq!(userns["nsmode"].as_str(), Some("nomap")); + assert!(userns.get("value").is_none(), "nomap should omit value"); + + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set for nomap" + ); + } + + #[test] + fn container_spec_omits_userns_when_unset() { + let sandbox = test_sandbox("no-userns-id", "no-userns-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + assert!( + spec.get("userns").is_none(), + "userns should not be set when unconfigured" + ); + assert!( + spec.get("idmappings").is_none(), + "idmappings should not be set when userns is unconfigured" + ); + } + + #[test] + fn container_spec_uses_bind_mount_for_supervisor_when_path_provided() { + let sandbox = test_sandbox("bind-sv-id", "bind-sv-name"); + let config = test_config(); + let image = resolve_image(&sandbox, &config); + let spec = build_container_spec_for_image( + &sandbox, + &config, + None, + None, + image, + image, + "", + Some(Path::new("/host/cache/openshell-sandbox")), + ) + .unwrap(); + + let image_volumes = spec["image_volumes"] + .as_array() + .expect("image_volumes should be an array"); + assert!( + !image_volumes + .iter() + .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), + "supervisor image volume should not be present when bind path is provided" + ); + + let mounts = spec["mounts"] + .as_array() + .expect("mounts should be an array"); + let sv_bind = mounts + .iter() + .find(|m| m["destination"].as_str() == Some(SUPERVISOR_BINARY_PATH)); + assert!( + sv_bind.is_some(), + "supervisor bind mount should be present at {SUPERVISOR_BINARY_PATH}" + ); + let sv_bind = sv_bind.unwrap(); + assert_eq!( + sv_bind["source"].as_str(), + Some("/host/cache/openshell-sandbox") + ); + assert_eq!(sv_bind["type"].as_str(), Some("bind")); + } + + #[test] + fn container_spec_uses_image_volume_when_no_bind_path() { + let sandbox = test_sandbox("imgvol-id", "imgvol-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let image_volumes = spec["image_volumes"] + .as_array() + .expect("image_volumes should be an array"); + assert!( + image_volumes + .iter() + .any(|v| v["destination"].as_str() == Some(SUPERVISOR_MOUNT_DIR)), + "supervisor image volume should be present by default" + ); + + let mounts = spec["mounts"] + .as_array() + .expect("mounts should be an array"); + assert!( + !mounts.iter().any( + |m| m["destination"].as_str() == Some(SUPERVISOR_BINARY_PATH) + && m["type"].as_str() == Some("bind") + ), + "supervisor bind mount should not be present by default" + ); + } } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index dd196f7992..a418a95b5c 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -11,7 +11,10 @@ use crate::watcher::{ }; use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; -use openshell_core::driver_utils::supervisor_image_should_refresh; +use openshell_core::driver_utils::{ + SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, + temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, +}; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, @@ -277,6 +280,7 @@ impl PodmanComputeDriver { config.validate_runtime_limits()?; config.validate_host_gateway_ip()?; config.validate_proxy_config()?; + config.validate_userns()?; let client = PodmanClient::new(socket_path); @@ -755,6 +759,18 @@ impl PodmanComputeDriver { return Err(e); } }; + let supervisor_bin_path = if userns_needs_extraction(self.config.userns.as_deref()) { + match extract_supervisor_bin(&self.client, &self.config).await { + Ok(path) => Some(path), + Err(e) => { + cleanup_created().await; + return Err(e); + } + } + } else { + None + }; + let spec = match container::build_container_spec_for_image( sandbox, &self.config, @@ -763,6 +779,7 @@ impl PodmanComputeDriver { image, &inspected_image.id, image_user, + supervisor_bin_path.as_deref(), ) { Ok(spec) => spec, Err(e) => { @@ -1092,6 +1109,97 @@ fn validate_rootless_local_callback_helper( ))) } +// ── Supervisor binary extraction (userns fallback) ───────────────────── + +async fn extract_supervisor_bin( + client: &PodmanClient, + config: &PodmanComputeConfig, +) -> Result { + let inspect = client + .inspect_image(&config.supervisor_image) + .await + .map_err(ComputeDriverError::from)?; + let digest = if inspect.id.is_empty() { + return Err(ComputeDriverError::Precondition(format!( + "supervisor image '{}' has no ID", + config.supervisor_image, + ))); + } else { + &inspect.id + }; + + let cache_path = + openshell_core::driver_utils::supervisor_cache_path("podman-supervisor", digest) + .map_err(ComputeDriverError::Precondition)?; + if cache_path.is_file() { + validate_linux_elf_binary(&cache_path).map_err(ComputeDriverError::Precondition)?; + info!( + cache_path = %cache_path.display(), + "Using cached supervisor binary" + ); + return Ok(cache_path); + } + + info!( + image = %config.supervisor_image, + cache_path = %cache_path.display(), + "Extracting supervisor binary from image" + ); + + let container_name = temp_extract_container_name(); + let spec = serde_json::json!({ + "image": config.supervisor_image, + "name": container_name, + "entrypoint": [SUPERVISOR_IMAGE_BINARY_PATH], + "command": [], + }); + client + .create_container(&spec) + .await + .map_err(ComputeDriverError::from)?; + + let result = extract_binary_from_container(client, &container_name, &cache_path).await; + + if let Err(err) = client.remove_container(&container_name, 0).await { + warn!( + container = container_name, + error = %err, + "Failed to remove supervisor extractor container" + ); + } + + result +} + +async fn extract_binary_from_container( + client: &PodmanClient, + container_name: &str, + cache_path: &Path, +) -> Result { + let tar_bytes = client + .copy_from_container(container_name, SUPERVISOR_IMAGE_BINARY_PATH) + .await + .map_err(ComputeDriverError::from)?; + + let binary_bytes = extract_first_tar_entry(&tar_bytes).map_err(|err| { + ComputeDriverError::Precondition(format!( + "failed to extract supervisor binary from tar: {err}" + )) + })?; + + write_cache_binary_atomic(cache_path, &binary_bytes) + .map_err(ComputeDriverError::Precondition)?; + validate_linux_elf_binary(cache_path).map_err(ComputeDriverError::Precondition)?; + Ok(cache_path.to_path_buf()) +} + +fn userns_needs_extraction(userns: Option<&str>) -> bool { + userns.is_some_and(|mode| { + let base = mode.split(':').next().unwrap_or(mode); + !base.eq_ignore_ascii_case("host") + }) +} + #[cfg(test)] mod tests { use super::*; @@ -2244,4 +2352,17 @@ mod tests { ); let _ = fs::remove_file(socket_path); } + + #[test] + fn userns_needs_extraction_cases() { + assert!(!userns_needs_extraction(None)); + assert!(!userns_needs_extraction(Some("host"))); + assert!(!userns_needs_extraction(Some("Host"))); + assert!(userns_needs_extraction(Some("auto"))); + assert!(userns_needs_extraction(Some("auto:size=65536"))); + assert!(userns_needs_extraction(Some("keep-id"))); + assert!(userns_needs_extraction(Some("keep-id:uid=1000"))); + assert!(userns_needs_extraction(Some("private"))); + assert!(userns_needs_extraction(Some("nomap"))); + } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 4a38643f38..cec5765357 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -133,6 +133,11 @@ struct Args { /// SSRF/`allowed_ips` validation no longer binds the connection. #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] sandbox_proxy_connect_by_hostname: Option, + + /// User namespace mode for sandbox containers (e.g. `auto`). + /// When unset, containers use the default user namespace. + #[arg(long, env = "OPENSHELL_PODMAN_USERNS")] + userns: Option, } #[tokio::main] @@ -168,6 +173,7 @@ async fn main() -> Result<()> { proxy_auth_file: args.sandbox_proxy_auth_file, proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, + userns: args.userns, ..PodmanComputeConfig::default() }) .await diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 2fd5717aec..c7e679cbdb 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -455,6 +455,8 @@ sandbox_pids_limit = 2048 # but increase process churn (each check spawns a conmon subprocess). # Set to 0 to disable health checks entirely. Default: 10. health_check_interval_secs = 10 +# User namespace mode for sandbox containers. Omit to use the default. +# userns = "auto" # Corporate forward proxy for sandbox egress. When set, the in-container # supervisor chains policy-approved TLS tunnels through this proxy with HTTP # CONNECT instead of dialing destinations directly. Plain-HTTP requests are