diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f7a1d9ee0c..2de05d409cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,6 +231,20 @@ jobs: mkdir -p "${openssl_cache_dir}" cp -R "${openssl_installs[0]}/." "${openssl_cache_dir}/" + - name: Test Windows container exec + if: runner.os == 'Windows' + shell: pwsh + run: ./tools/ci/windows-exec-tests.ps1 + + - name: Upload Windows container exec test results + if: always() && runner.os == 'Windows' && env.WINDOWS_EXEC_RESULTS != '' + uses: actions/upload-artifact@v4 + with: + name: windows-container-exec-results + path: ${{ env.WINDOWS_EXEC_RESULTS }} + if-no-files-found: error + retention-days: 14 + - name: Package build artifacts shell: bash run: | diff --git a/Cargo.lock b/Cargo.lock index 0020e4b5c2b..e06e7f60bee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3109,6 +3109,13 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hosted-auth-test" +version = "0.0.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "http" version = "1.3.1" @@ -7919,6 +7926,7 @@ name = "spacetimedb-cli" version = "2.10.0" dependencies = [ "anyhow", + "axum", "base64 0.21.7", "bytes", "cargo_metadata", @@ -7953,20 +7961,24 @@ dependencies = [ "rolldown_common", "rolldown_error", "rolldown_utils", + "rustix 1.1.2", "rustyline", "serde", "serde_json", "serde_with", + "sha2", "spacetimedb-auth", "spacetimedb-client-api-messages", "spacetimedb-codegen", "spacetimedb-data-structures", "spacetimedb-fs-utils", "spacetimedb-lib", + "spacetimedb-oci", "spacetimedb-paths", "spacetimedb-schema", "syntect", "tabled", + "tar", "tempfile", "termcolor", "termtree", @@ -7974,8 +7986,11 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-tungstenite 0.27.0", + "tokio-util", "toml 0.8.23", "toml_edit 0.22.27", + "url", + "uuid", "walkdir", "wasmbin", "webbrowser", @@ -8200,6 +8215,7 @@ dependencies = [ "tracing-subscriber", "tracing-tracy", "url", + "uuid", "v8", "wasmtime", "wasmtime-internal-fiber", @@ -8420,6 +8436,7 @@ dependencies = [ "spacetimedb-metrics", "spacetimedb-primitives", "spacetimedb-sats", + "thiserror 1.0.69", ] [[package]] @@ -8442,6 +8459,20 @@ dependencies = [ "prometheus", ] +[[package]] +name = "spacetimedb-oci" +version = "2.10.0" +dependencies = [ + "anyhow", + "flate2", + "serde", + "serde_json", + "sha2", + "spacetimedb-lib", + "tar", + "zstd", +] + [[package]] name = "spacetimedb-paths" version = "2.10.0" @@ -8644,6 +8675,9 @@ dependencies = [ "native-tls", "once_cell", "prometheus", + "reqwest", + "serde", + "serde_json", "shlex", "spacetimedb-client-api-messages", "spacetimedb-data-structures", @@ -8817,11 +8851,13 @@ dependencies = [ "serde", "serde_json", "serial_test", + "spacetimedb-auth", "spacetimedb-cli", "spacetimedb-client-api", "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-data-structures", + "spacetimedb-datastore", "spacetimedb-guard", "spacetimedb-lib", "spacetimedb-paths", @@ -10205,6 +10241,7 @@ checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", "js-sys", + "serde", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index ba682e45ba5..f0e3c77b2f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/fs-utils", "crates/index-scan-gate", "crates/lib", + "crates/oci", "crates/metrics", "crates/paths", "crates/pg", @@ -46,6 +47,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/hosted-auth-test", "modules/environment-test", "modules/invocation-flags-test", "templates/basic-rs/spacetimedb", diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs new file mode 100644 index 00000000000..8601448bc87 --- /dev/null +++ b/crates/auth/src/hosted.rs @@ -0,0 +1,393 @@ +//! Target-bound credentials for a database's hosted container. +//! +//! Decoded claims are untrusted. Only signature verification against an explicitly +//! configured platform issuer and comparison with an authoritative instance/grant +//! binding can produce [`VerifiedHostedAuth`]. Receiving hosts must additionally +//! recheck their durable target fence at each transaction and subscription admission. + +use crate::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; +use anyhow::{bail, ensure, Context}; +use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::Identity; +use std::fmt; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub const HOSTED_TOKEN_KIND: &str = "spacetimedb_hosted_v1"; +pub const HOSTED_TOKEN_TYPE: &str = "spacetimedb-hosted+jwt"; +pub const MAX_HOSTED_TOKEN_LIFETIME: Duration = Duration::from_secs(30); +pub const MAX_HOSTED_TOKEN_BYTES: usize = 8192; + +/// Wire claims, deliberately distinct from ordinary issuer/subject-derived Identity claims. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostedTokenClaims { + pub kind: Box, + #[serde(rename = "iss")] + pub issuer: Box, + #[serde(rename = "sub")] + pub subject: Box, + #[serde(with = "identity_hex")] + pub source_database: Identity, + /// A scalar, canonical database Identity. Lists and database names are not accepted. + #[serde(rename = "aud", with = "identity_hex")] + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + #[serde(rename = "iat")] + pub issued_at: u64, + #[serde(rename = "exp")] + pub expires_at: u64, + #[serde(rename = "jti")] + pub token_id: Box, +} + +/// Trusted input obtained from current source registration, placement and target grant state. +/// Never construct this by copying the incoming token's claims. Admission must be open, +/// and the assigned node/incarnation and required module capability must already be checked. +#[derive(Clone, Copy, Debug)] +pub struct HostedTokenBinding { + pub source_database: Identity, + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + pub lease_expires_at: SystemTime, +} + +/// Authentication proof. It cannot be deserialized or constructed from decoded claims. +#[derive(Clone)] +pub struct VerifiedHostedAuth { + claims: HostedTokenClaims, + // Local lifetime state only. Never serialized into signed claims. + monotonic_deadline: Instant, +} + +impl fmt::Debug for VerifiedHostedAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VerifiedHostedAuth") + .field("source_database", &self.source_database()) + .field("target_database", &self.target_database()) + .field("generation", &self.generation()) + .field("grant_revision", &self.grant_revision()) + .field("expires_at", &self.expires_at()) + .finish_non_exhaustive() + } +} + +impl VerifiedHostedAuth { + pub fn source_database(&self) -> Identity { + self.claims.source_database + } + pub fn target_database(&self) -> Identity { + self.claims.target_database + } + pub fn generation(&self) -> u64 { + self.claims.generation + } + pub fn grant_revision(&self) -> u64 { + self.claims.grant_revision + } + pub fn issued_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.issued_at) + } + pub fn expires_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.expires_at) + } + pub fn token_id(&self) -> &str { + &self.claims.token_id + } + pub fn issuer(&self) -> &str { + &self.claims.issuer + } + pub fn is_internal(&self) -> bool { + self.source_database() == self.target_database() + } + + /// Expiry has no positive leeway. This does not replace generation/grant fencing. + pub fn check_at(&self, now: SystemTime) -> anyhow::Result<()> { + self.check_at_clocks(now, Instant::now()) + } + + fn check_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> anyhow::Result<()> { + ensure!(now >= self.issued_at(), "hosted credential is not yet valid"); + ensure!( + now < self.expires_at() && monotonic_now < self.monotonic_deadline, + "hosted credential expired" + ); + Ok(()) + } + + /// Cap the local lifetime using a fresh authority confirmation. The caller + /// records `confirmation_started` before sending the request and obtains + /// `confirmed_time` from the successful authoritative response. Charging the + /// entire round trip against the signed remaining lifetime is conservative. + /// Neither a receiving clock behind authority nor a later confirmation can + /// extend this proof. The original signed claims are preserved exactly. + pub fn constrain_expiration( + mut self, + confirmed_time: SystemTime, + confirmation_started: Instant, + ) -> anyhow::Result { + let now = Instant::now(); + ensure!(confirmation_started <= now, "invalid hosted confirmation clock"); + self.check_at_clocks(confirmed_time, now)?; + let remaining = self.expires_at().duration_since(confirmed_time)?; + let confirmed_deadline = confirmation_started + .checked_add(remaining) + .context("hosted confirmation deadline overflow")?; + self.monotonic_deadline = self.monotonic_deadline.min(confirmed_deadline); + self.check_at_clocks(confirmed_time, now)?; + Ok(self) + } + + /// Remaining connection lifetime, bounded by both signed wall-clock expiry + /// and the already captured local deadline. Repeated calls never reset it. + pub fn remaining_lifetime(&self, now: SystemTime) -> Duration { + self.remaining_lifetime_at_clocks(now, Instant::now()) + } + + fn remaining_lifetime_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> Duration { + if self.check_at_clocks(now, monotonic_now).is_err() { + return Duration::ZERO; + } + self.expires_at() + .duration_since(now) + .unwrap_or_default() + .min(self.monotonic_deadline.saturating_duration_since(monotonic_now)) + } + + pub fn into_connection_auth(self) -> anyhow::Result { + // Keep the actual claims, including the source/target/generation restrictions. + // Normalizing JSON whitespace does not alter any signed claim values. + let jwt_payload = serde_json::to_string(&self.claims)?.into_boxed_str(); + let mut extra = serde_json::to_value(&self.claims)?; + let extra = extra.as_object_mut().expect("hosted claims serialize as an object"); + for key in ["iss", "sub", "aud", "iat", "exp"] { + extra.remove(key); + } + let claims = SpacetimeIdentityClaims { + identity: self.source_database(), + subject: self.claims.subject.clone(), + issuer: self.claims.issuer.clone(), + audience: [self.target_database().to_hex().to_string().into_boxed_str()].into(), + iat: self.issued_at(), + exp: Some(self.expires_at()), + extra: Some( + extra + .iter() + .map(|(key, value)| (key.clone().into_boxed_str(), value.clone())) + .collect(), + ), + }; + Ok(ConnectionAuthCtx { + claims, + jwt_payload, + hosted: Some(self), + }) + } +} + +/// Classifies the reserved namespace only. A positive result grants no authority. +/// All reserved versions are rejected by ordinary OIDC validation and token exchange. +pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token(token, is_reserved_hosted_type, is_reserved_hosted_kind) +} + +/// Operational container proofs are never client Identity credentials. Reserve +/// their entire versioned namespace so a lease/registry proof cannot enter +/// OIDC discovery, ordinary JWT validation, or the Identity token exchange. +pub fn has_reserved_platform_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token( + token, + |kind| is_reserved_hosted_type(kind) || kind.starts_with("spacetimedb-container-"), + |kind| is_reserved_hosted_kind(kind) || kind.starts_with("spacetimedb_container_"), + ) +} + +fn classify_reserved_token( + token: &str, + reserved_type: impl FnOnce(&str) -> bool, + reserved_kind: impl FnOnce(&str) -> bool, +) -> anyhow::Result { + let header = decode_header(token)?; + if header.typ.as_deref().is_some_and(reserved_type) { + return Ok(true); + } + let data = jsonwebtoken::dangerous::insecure_decode::(token)?; + Ok(data + .claims + .get("kind") + .and_then(serde_json::Value::as_str) + .is_some_and(reserved_kind)) +} + +pub fn is_reserved_hosted_kind(kind: &str) -> bool { + kind.starts_with("spacetimedb_hosted_") +} +fn is_reserved_hosted_type(kind: &str) -> bool { + kind.starts_with("spacetimedb-hosted") +} + +/// Decode routing hints only, never authentication. The caller must use these hints to +/// find trusted registration/grant state and then call [`verify_hosted_token`]. +pub fn unverified_hosted_token_claims(token: &str) -> anyhow::Result { + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + Ok(jsonwebtoken::dangerous::insecure_decode::(token)?.claims) +} + +/// Verify against a configured key and issuer, never a JWT-supplied key or JWKS URL. +/// `binding` must be authoritative state for that issuer's registered source database. +pub fn verify_hosted_token( + token: &str, + public_key: &DecodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result { + let verification_started = Instant::now(); + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let header = decode_header(token)?; + ensure!(header.alg == Algorithm::ES256, "hosted credential requires ES256"); + ensure!( + header.typ.as_deref() == Some(HOSTED_TOKEN_TYPE), + "invalid hosted credential type" + ); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_required_spec_claims(&["iss", "sub", "aud", "exp"]); + validation.set_issuer(&[trusted_issuer]); + validation.set_audience(&[binding.target_database.to_hex().to_string()]); + validation.leeway = 0; + // Check time below against the caller's trusted clock, including exact expiry equality. + validation.validate_exp = false; + let claims = decode::(token, public_key, &validation)?.claims; + validate_claims(&claims, trusted_issuer, binding, now)?; + let remaining = (UNIX_EPOCH + Duration::from_secs(claims.expires_at)).duration_since(now)?; + let monotonic_deadline = verification_started + .checked_add(remaining) + .context("hosted credential deadline overflow")?; + let proof = VerifiedHostedAuth { + claims, + monotonic_deadline, + }; + proof.check_at(now)?; + Ok(proof) +} + +/// Mint from the broker's authoritative binding, with no guest-selected sender or generation. +pub fn sign_hosted_token( + private_key: &EncodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, + expires_at: SystemTime, + token_id: &str, +) -> anyhow::Result { + let claims = HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: trusted_issuer.into(), + subject: binding.source_database.to_hex().to_string().into_boxed_str(), + source_database: binding.source_database, + target_database: binding.target_database, + generation: binding.generation, + grant_revision: binding.grant_revision, + issued_at: unix_seconds(now)?, + expires_at: unix_seconds(expires_at)?, + token_id: token_id.into(), + }; + validate_claims(&claims, trusted_issuer, binding, now)?; + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + Ok(jsonwebtoken::encode(&header, &claims, private_key)?) +} + +fn validate_claims( + claims: &HostedTokenClaims, + issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result<()> { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + claims.kind.as_ref() == HOSTED_TOKEN_KIND, + "unsupported hosted credential kind" + ); + ensure!(claims.issuer.as_ref() == issuer, "untrusted hosted credential issuer"); + ensure!( + claims.source_database == binding.source_database, + "hosted credential source mismatch" + ); + ensure!( + claims.target_database == binding.target_database, + "hosted credential target mismatch" + ); + ensure!( + claims.subject.as_ref() == claims.source_database.to_hex().as_str(), + "hosted credential subject mismatch" + ); + ensure!( + claims.generation == binding.generation, + "hosted credential generation mismatch" + ); + ensure!( + claims.grant_revision == binding.grant_revision, + "hosted credential grant revision mismatch" + ); + ensure!( + !claims.token_id.is_empty() + && claims.token_id.len() <= 128 + && claims + .token_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'), + "invalid hosted credential token ID" + ); + let Some(lifetime) = claims.expires_at.checked_sub(claims.issued_at) else { + bail!("invalid hosted credential lifetime") + }; + ensure!( + lifetime > 0 && lifetime <= MAX_HOSTED_TOKEN_LIFETIME.as_secs(), + "hosted credential lifetime exceeds limit" + ); + let issued_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.issued_at)) + .context("invalid hosted issue time")?; + let expires_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.expires_at)) + .context("invalid hosted expiry")?; + ensure!( + issued_at <= now && now < expires_at, + "hosted credential outside validity interval" + ); + ensure!( + expires_at <= binding.lease_expires_at, + "hosted credential exceeds confirmed lease" + ); + Ok(()) +} + +fn unix_seconds(time: SystemTime) -> anyhow::Result { + Ok(time.duration_since(UNIX_EPOCH)?.as_secs()) +} + +mod identity_hex { + use super::*; + pub fn serialize(value: &Identity, serializer: S) -> Result { + serializer.serialize_str(value.to_hex().as_str()) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(serde::de::Error::custom( + "expected canonical 64-character lowercase database Identity", + )); + } + Identity::from_hex(value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +#[path = "hosted/expiration_tests.rs"] +mod expiration_tests; diff --git a/crates/auth/src/hosted/expiration_tests.rs b/crates/auth/src/hosted/expiration_tests.rs new file mode 100644 index 00000000000..1624bfc474f --- /dev/null +++ b/crates/auth/src/hosted/expiration_tests.rs @@ -0,0 +1,136 @@ +use super::*; + +fn proof() -> (VerifiedHostedAuth, SystemTime, Instant) { + let wall = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let monotonic = Instant::now(); + ( + VerifiedHostedAuth { + claims: HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: "platform.test".into(), + subject: Identity::ZERO.to_hex().to_string().into(), + source_database: Identity::ZERO, + target_database: Identity::ZERO, + generation: 1, + grant_revision: 2, + issued_at: 1_700_000_000, + expires_at: 1_700_000_030, + token_id: "expiration-test".into(), + }, + monotonic_deadline: monotonic + Duration::from_secs(30), + }, + wall, + monotonic, + ) +} + +#[test] +fn receiving_clock_behind_control_cannot_extend_confirmed_lifetime() { + let (proof, wall, monotonic) = proof(); + let request_started = monotonic - Duration::from_secs(2); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), request_started) + .unwrap(); + let deadline = request_started + Duration::from_secs(5); + assert_eq!(proof.monotonic_deadline, deadline); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(3), deadline - Duration::from_nanos(1)) + .is_ok()); + assert!(proof.check_at_clocks(wall + Duration::from_secs(3), deadline).is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(3), deadline), + Duration::ZERO + ); +} + +#[test] +fn confirmation_round_trip_consumes_remaining_signed_lifetime() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic - Duration::from_secs(6)) + .is_err()); +} + +#[test] +fn later_confirmation_cannot_extend_a_previous_deadline() { + let (proof, wall, monotonic) = proof(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let first_deadline = proof.monotonic_deadline; + // Even a second response with an earlier authority timestamp cannot extend + // the stricter deadline already held by this authentication proof. + let proof = proof + .constrain_expiration(wall + Duration::from_secs(20), Instant::now()) + .unwrap(); + assert_eq!(proof.monotonic_deadline, first_deadline); +} + +#[test] +fn backward_wall_clock_does_not_restart_the_local_lifetime() { + let (proof, wall, monotonic) = proof(); + let later = monotonic + Duration::from_secs(29); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(1), later), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(1), monotonic + Duration::from_secs(30)) + .is_err()); +} + +#[test] +fn wall_clock_expiration_remains_an_independent_limit() { + let (proof, wall, monotonic) = proof(); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(29), monotonic), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall - Duration::from_secs(1), monotonic), + Duration::ZERO + ); +} + +#[test] +fn invalid_confirmation_clocks_are_rejected() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .clone() + .constrain_expiration(wall - Duration::from_secs(1), monotonic) + .is_err()); + assert!(proof + .clone() + .constrain_expiration(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert!(proof + .constrain_expiration(wall, Instant::now() + Duration::from_secs(60)) + .is_err()); +} + +#[test] +fn cloning_and_connection_conversion_preserve_the_cap_and_signed_claims() { + let (proof, wall, monotonic) = proof(); + let original_claims = serde_json::to_value(&proof.claims).unwrap(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let deadline = proof.monotonic_deadline; + let cloned = proof.clone(); + let connection = proof.into_connection_auth().unwrap(); + let connection = connection.clone(); + let retained = connection.hosted.as_ref().unwrap(); + assert_eq!(cloned.monotonic_deadline, deadline); + assert_eq!(retained.monotonic_deadline, deadline); + assert_eq!(connection.claims.exp, Some(wall + Duration::from_secs(30))); + assert_eq!( + serde_json::from_str::(&connection.jwt_payload).unwrap(), + original_claims + ); + assert!(retained + .check_at_clocks(wall + Duration::from_secs(1), deadline) + .is_err()); +} diff --git a/crates/auth/src/identity.rs b/crates/auth/src/identity.rs index 41b38d00cb7..380945f0bfb 100644 --- a/crates/auth/src/identity.rs +++ b/crates/auth/src/identity.rs @@ -7,10 +7,20 @@ use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::Identity; use std::time::SystemTime; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ConnectionAuthCtx { pub claims: SpacetimeIdentityClaims, pub jwt_payload: Box, + pub hosted: Option, +} + +impl std::fmt::Debug for ConnectionAuthCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionAuthCtx") + .field("identity", &self.claims.identity) + .field("hosted", &self.hosted) + .finish_non_exhaustive() + } } impl TryFrom for ConnectionAuthCtx { @@ -20,6 +30,7 @@ impl TryFrom for ConnectionAuthCtx { Ok(ConnectionAuthCtx { claims, jwt_payload: payload.into(), + hosted: None, }) } } @@ -103,6 +114,15 @@ impl TryInto for IncomingClaims { type Error = anyhow::Error; fn try_into(self) -> anyhow::Result { + if self + .extra + .as_ref() + .and_then(|extra| extra.get("kind")) + .and_then(serde_json::Value::as_str) + .is_some_and(crate::hosted::is_reserved_hosted_kind) + { + anyhow::bail!("hosted credentials require dedicated target-bound validation"); + } // The issuer and subject must be less than 128 bytes. if self.issuer.len() > 128 { return Err(anyhow::anyhow!("Issuer too long: {:?}", self.issuer)); diff --git a/crates/auth/src/lib.rs b/crates/auth/src/lib.rs index db53a0c9064..b8b0976f40d 100644 --- a/crates/auth/src/lib.rs +++ b/crates/auth/src/lib.rs @@ -1 +1,2 @@ +pub mod hosted; pub mod identity; diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index ddf546c296e..68132fd9573 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -255,3 +255,147 @@ by the host in a submodule fail at runtime; ordinary helpers retain their caller scope. Values are private, durable database configuration for secrets and other settings. Database owners and authorized collaborators can read them; module code can expose them through its own outputs. + +### Node.js Authorization transport + +Node.js clients can send a token directly in the WebSocket `Authorization` +header by selecting the explicit Node transport. Install its optional peer: + +```sh +npm install ws +``` + +```ts +import { openNodeWebSocket } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +function connect( + serverUri: string, + databaseIdentity: string, + freshToken: string +) { + return DbConnection.builder() + .withUri(serverUri) + .withDatabaseName(databaseIdentity) + .withToken(freshToken) + .withWSFn(openNodeWebSocket) + .onConnect(connection => { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + }) + .build(); +} +``` + +Use a trusted `wss://` server in deployed clients. This transport sends the +provided token only in the upgrade request header, preserves the SDK connection +ID and subscription options, and refuses redirects. It does not exchange the +token through the generic identity endpoint or place credentials in the URL. +The ordinary browser transport is unchanged. + +This factory transports caller-supplied credentials; it does not discover or +renew container credentials. Obtain a fresh target-specific credential for each +new connection and recreate subscriptions when reconnecting. Unconfirmed +reducer and procedure calls may already have committed and must not be +replayed automatically. Do not log or persist credentials. + +`connection.disconnect()` initiates closure. If retaining the adapter from a +custom `withWSFn` wrapper, `await adapter.shutdown()` waits for actual transport +closure. A stalled upgrade or a peer that ignores the close handshake is +terminated after five seconds. + +### Credentials inside a container + +The Node entry point also reads the platform's discovery variables and obtains +short-lived credentials from the local broker: + +```ts +import { Container, openNodeWebSocket } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +const container = Container.fromEnvironment(); +const credential = await container.tokenFor(); +const connection = DbConnection.builder() + .withUri(container.serverUri) + .withDatabaseName(credential.target.toHexString()) + .withToken(credential.value) + .withWSFn(openNodeWebSocket) + .onConnect(connection => { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + }) + .build(); +``` + +`tokenFor()` targets the container's own database. Pass another database's +`Identity` to request a credential for it, then connect to that database's trusted +server. The broker determines whether access is allowed. For an HTTP request, +send `credential.value` in the `Authorization: Bearer ...` header. + +`Container.fromEnvironment()` reads only `SPACETIMEDB_DATABASE_IDENTITY`, +`SPACETIMEDB_SERVER_URI`, and `SPACETIMEDB_CREDENTIAL_BROKER`. Missing or invalid +values fail explicitly. `new Container({ databaseIdentity, serverUri, +credentialBroker })` accepts explicit discovery configuration. The local broker +uses an absolute Unix socket or numeric loopback HTTP address; requests neither +follow redirects nor use saved credentials or proxy settings. + +Each request opens a fresh connection and completes within five seconds, with +an optional `AbortSignal` as the second argument to `tokenFor`. Tokens last at +most 30 seconds. `expiresAt` and `remainingLifetimeMs` expose their remaining +validity; logging or serializing the token object redacts its value. Rebuild the +database connection with a fresh token and restore subscriptions on expiry or +disconnection. This helper does not manage connection renewal or replay calls. + +### Managed container connections in Node.js + +`ContainerSession` owns credential renewal and successive connections. Pass the +unmodified generated `DbConnection` class so it can create a fresh builder, +cache, and subscriptions for every generation: + +```ts +import { Container, ContainerSession } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +const session = new ContainerSession(DbConnection, { + container: Container.fromEnvironment(), + onConnect(connection, { generation }) { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + console.log('Connected generation', generation); + }, +}); +const pause = new AbortController(); +process.once('SIGTERM', () => pause.abort()); +try { + await session.run(pause.signal); +} finally { + await session.shutdown(); +} +``` + +The owner pins the database Identity and server before requesting credentials. +Optional `target: Identity` and `serverUri` select another database and its +trusted server. It verifies the connecting sender against the container's +Identity before calling `onConnect`. Setup and event callbacks must be +synchronous. Application asynchronous work must have its own cancellation and +error handling. + +Renewal requests a fresh token before expiry. Both the returned expiry and its +monotonic lifetime must extend materially before the owner rotates connections; +the same lease expiry does not cause repeated rotations. A rejected credential +is terminal. Transient broker failures retry while the current credential +remains valid, and expiry seals the old generation even if a refresh stalls. +The old WebSocket is closed and joined before a new one is opened. This causes a +brief interruption and requires restoring subscriptions in every `onConnect`. + +Aborting `run(signal)` immediately seals that generation, then joins its broker +request and WebSocket before resolving. Calling `run` again resumes with fresh +credentials. `shutdown()` is permanent and also awaits cleanup. Concurrent +`run` calls are rejected. Keep and await the owner; discarding its promise does +not cancel it. + +No reducer or procedure is replayed. A pending call receives +`ContainerSessionCallError`: `outcome: 'not_sent'` means it was never handed to +the transport, while `outcome: 'unknown'` means no confirmed result was received +after handoff. A confirmed result wins a later shutdown. Generation +`disconnected` events also identify the interruption and warn about unconfirmed +calls. Retained old connections reject new calls as `not_sent`, reject new +subscriptions, and release their callbacks. Their cache is an old snapshot; +the next generation starts with a separate empty cache. diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index db598440673..0a5bfe2f2f6 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -101,6 +101,12 @@ "import": "./dist/solid/index.mjs", "require": "./dist/solid/index.cjs", "default": "./dist/solid/index.mjs" + }, + "./sdk/node": { + "types": "./dist/sdk/node/index.d.ts", + "import": "./dist/sdk/node/index.mjs", + "require": "./dist/sdk/node/index.cjs", + "default": "./dist/sdk/node/index.mjs" } }, "size-limit": [ @@ -189,7 +195,8 @@ "pure-rand": "^7.0.1", "safe-stable-stringify": "^2.5.0", "statuses": "^2.0.2", - "url-polyfill": "^1.1.14" + "url-polyfill": "^1.1.14", + "@types/ws": "^8.18.1" }, "peerDependencies": { "@angular/core": ">=17.0.0", @@ -198,7 +205,8 @@ "solid-js": "^1.6.0", "svelte": "^4.0.0 || ^5.0.0", "undici": "^6.19.2", - "vue": "^3.3.0" + "vue": "^3.3.0", + "ws": "^8.18.3" }, "peerDependenciesMeta": { "@tanstack/react-query": { @@ -221,6 +229,9 @@ }, "@angular/core": { "optional": true + }, + "ws": { + "optional": true } }, "devDependencies": { diff --git a/crates/bindings-typescript/src/sdk/db_connection_impl.ts b/crates/bindings-typescript/src/sdk/db_connection_impl.ts index 12dfca7afbf..f3d7aa7034e 100644 --- a/crates/bindings-typescript/src/sdk/db_connection_impl.ts +++ b/crates/bindings-typescript/src/sdk/db_connection_impl.ts @@ -1,3 +1,9 @@ +import { + INTERNAL_MANAGED_SESSION, + INTERNAL_CLEAR_TABLE_CALLBACKS, + ContainerSessionCallError, + type ManagedSessionLifecycle, +} from './managed_session_lifecycle'; import { ConnectionId, ProductBuilder, ProductType } from '../'; import { AlgebraicType, type ComparablePrimitive } from '../'; import BinaryReader from '../lib/binary_reader.ts'; @@ -274,6 +280,87 @@ export class DbConnectionImpl >(); #reducerCallInfo = new Map(); #procedureCallbacks = new Map(); + #managedCalls?: Map< + number, + { reject: (error: Error) => void; sent: boolean } + >; + #managedQueuedCalls = new Map(); + #managedTerminalError?: Error; + + [INTERNAL_MANAGED_SESSION](): ManagedSessionLifecycle { + return { + enable: () => { + if ( + this.#managedCalls || + this.#managedTerminalError || + this.#outboundQueue.length || + this.#reducerCallbacks.size || + this.#procedureCallbacks.size || + this.#subscriptionManager.subscriptions.size + ) + throw new Error('Managed boundary requires an unused connection'); + this.#managedCalls = new Map(); + }, + seal: error => { + if (!this.#managedCalls || this.#managedTerminalError) return; + // Seal first, before any subscription callback can attempt another call. + this.#managedTerminalError = error; + this.isActive = false; + this.isDisconnectRequested = true; + this.token = undefined; + this.#outboundQueue.length = 0; + this.#inboundQueue.length = 0; + this.#inboundQueueOffset = 0; + this.#managedQueuedCalls.clear(); + const pending = [...this.#managedCalls.values()]; + this.#managedCalls.clear(); + this.#reducerCallbacks.clear(); + this.#procedureCallbacks.clear(); + this.#reducerCallInfo.clear(); + this.#emitter.clear(); + for (const table of this.clientCache.tables.values()) + table[INTERNAL_CLEAR_TABLE_CALLBACKS](); + const subscriptions = [ + ...this.#subscriptionManager.subscriptions.values(), + ]; + this.#subscriptionManager.subscriptions.clear(); + for (const call of pending) + call.reject( + new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent') + ); + const errorContext: ErrorContextInterface = { + ...this.#makeEventContext({ + id: this.#nextEventId(), + tag: 'Error', + value: error, + }), + event: error, + }; + let callbackFailed = false; + for (const { emitter } of subscriptions) { + try { + emitter.emit('error', errorContext, error); + } catch { + callbackFailed = true; + } finally { + emitter.clear(); + } + } + if (callbackFailed) + throw new Error('Managed subscription callback failed'); + }, + }; + } + + #markManagedCallSent(message: Uint8Array): void { + const requestId = this.#managedQueuedCalls.get(message); + this.#managedQueuedCalls.delete(message); + if (requestId !== undefined) { + const call = this.#managedCalls?.get(requestId); + if (call) call.sent = true; + } + } + #rowDeserializers: Record>; #rowIdMetadata: Record< string, @@ -560,6 +647,7 @@ export class DbConnectionImpl >, querySql: string[] ): number { + if (this.#managedTerminalError) throw this.#managedTerminalError; const querySetId = this.#getNextQueryId(); this.#subscriptionManager.subscriptions.set(querySetId, { handle, @@ -715,6 +803,7 @@ export class DbConnectionImpl #flushOutboundQueueV2(wsResolved: WebSocketAdapter): void { const pending = this.#outboundQueue.splice(0); for (const message of pending) { + this.#markManagedCallSent(message); wsResolved.send(message); } } @@ -731,6 +820,8 @@ export class DbConnectionImpl this.#outboundQueue, MAX_V3_OUTBOUND_FRAME_BYTES ); + for (let index = 0; index < batchSize; index++) + this.#markManagedCallSent(this.#outboundQueue[index]); wsResolved.send( encodeClientMessagesV3( this.#clientFrameEncoder, @@ -786,35 +877,51 @@ export class DbConnectionImpl #clientMessageEncoder = new BinaryWriter(1024); #sendEncodedMessage( encoded: Uint8Array, - describe: () => string + describe: () => string, + managedRequestId?: number ): void { + if (this.#managedTerminalError) throw this.#managedTerminalError; stdbLogger('trace', describe); if (this.ws && this.isActive) { if (this.#negotiatedWsProtocol === V2_WS_PROTOCOL) { if (this.#outboundQueue.length) this.#flushOutboundQueue(this.ws); + const call = + managedRequestId === undefined + ? undefined + : this.#managedCalls?.get(managedRequestId); + if (call) call.sent = true; this.ws.send(encoded); return; } - this.#outboundQueue.push(encoded.slice()); + const queued = encoded.slice(); + this.#outboundQueue.push(queued); + if (this.#managedCalls && managedRequestId !== undefined) + this.#managedQueuedCalls.set(queued, managedRequestId); this.#scheduleOutboundFlush(); } else { // Use slice() to copy, in case the clientMessageEncoder's buffer gets reused // before the connection opens or before a v3 microbatch flush runs. - this.#outboundQueue.push(encoded.slice()); + const queued = encoded.slice(); + this.#outboundQueue.push(queued); + if (this.#managedCalls && managedRequestId !== undefined) + this.#managedQueuedCalls.set(queued, managedRequestId); } } - #sendMessage(message: ClientMessage): void { + #sendMessage(message: ClientMessage, managedRequestId?: number): void { const writer = this.#clientMessageEncoder; writer.clear(); ClientMessage.serialize(writer, message); const encoded = writer.getBuffer(); const isLive = !!(this.ws && this.isActive); - this.#sendEncodedMessage(encoded, () => - isLive - ? `Sending message to server: ${stringify(message)}` - : `Queuing message to server: ${stringify(message)}` + this.#sendEncodedMessage( + encoded, + () => + isLive + ? `Sending message to server: ${stringify(message)}` + : `Queuing message to server: ${stringify(message)}`, + managedRequestId ); } @@ -833,7 +940,8 @@ export class DbConnectionImpl const encoded = writer.getBuffer(); this.#sendEncodedMessage( encoded, - () => `Sending reducer call message to server: requestId=${requestId}` + () => `Sending reducer call message to server: requestId=${requestId}`, + requestId ); } @@ -852,7 +960,8 @@ export class DbConnectionImpl const encoded = writer.getBuffer(); this.#sendEncodedMessage( encoded, - () => `Sending procedure call message to server: requestId=${requestId}` + () => `Sending procedure call message to server: requestId=${requestId}`, + requestId ); } @@ -926,11 +1035,13 @@ export class DbConnectionImpl () => `Calling ${callbacks.length} triggered row callbacks` ); for (const callback of callbacks) { + if (this.#managedTerminalError) return; callback.cb(); } } #processServerMessage(serverMessage: ServerMessage): void { + if (this.#managedTerminalError) return; stdbLogger( 'trace', () => `Processing server message: ${stringify(serverMessage)}` @@ -1054,6 +1165,14 @@ export class DbConnectionImpl } case 'ReducerResult': { const { requestId, result } = serverMessage.value; + // A received result wins a later managed seal from a row callback. + // Promise reactions still run after this synchronous cache update. + if (this.#managedCalls?.has(requestId)) { + this.#managedCalls.delete(requestId); + const confirmed = this.#reducerCallbacks.get(requestId); + this.#reducerCallbacks.delete(requestId); + confirmed?.(result); + } if (result.tag === 'Ok') { const reducerInfo = this.#reducerCallInfo.get(requestId); @@ -1086,6 +1205,7 @@ export class DbConnectionImpl this.#reducerCallInfo.delete(requestId); const cb = this.#reducerCallbacks.get(requestId); this.#reducerCallbacks.delete(requestId); + this.#managedCalls?.delete(requestId); cb?.(result); break; } @@ -1097,6 +1217,7 @@ export class DbConnectionImpl : { tag: 'Err', value: status.value }; const cb = this.#procedureCallbacks.get(requestId); this.#procedureCallbacks.delete(requestId); + this.#managedCalls?.delete(requestId); cb?.(result); break; } @@ -1200,9 +1321,20 @@ export class DbConnectionImpl argsBuffer: Uint8Array, reducerArgs?: object ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); - this.#sendCallReducerMessage(requestId, encodedReducerName, argsBuffer); + this.#managedCalls?.set(requestId, { reject, sent: false }); + try { + this.#sendCallReducerMessage(requestId, encodedReducerName, argsBuffer); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } if (reducerArgs) { this.#reducerCallInfo.set(requestId, { name: reducerName, @@ -1235,15 +1367,26 @@ export class DbConnectionImpl argsBuffer: Uint8Array, reducerArgs?: object ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); + this.#managedCalls?.set(requestId, { reject, sent: false }); const message = ClientMessage.CallReducer({ reducer: reducerName, args: argsBuffer, requestId, flags: 0, }); - this.#sendMessage(message); + try { + this.#sendMessage(message, requestId); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } if (reducerArgs) { this.#reducerCallInfo.set(requestId, { name: reducerName, @@ -1316,9 +1459,24 @@ export class DbConnectionImpl encodedProcedureName: Uint8Array, argsBuffer: Uint8Array ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); - this.#sendCallProcedureMessage(requestId, encodedProcedureName, argsBuffer); + this.#managedCalls?.set(requestId, { reject, sent: false }); + try { + this.#sendCallProcedureMessage( + requestId, + encodedProcedureName, + argsBuffer + ); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } this.#procedureCallbacks.set(requestId, result => { if (result.tag === 'Ok') { resolve(result.value); @@ -1333,8 +1491,11 @@ export class DbConnectionImpl procedureName: string, argsBuffer: Uint8Array ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); + this.#managedCalls?.set(requestId, { reject, sent: false }); const message = ClientMessage.CallProcedure({ procedure: procedureName, args: argsBuffer, @@ -1342,7 +1503,15 @@ export class DbConnectionImpl // reserved for future use - 0 is the only valid value flags: 0, }); - this.#sendMessage(message); + try { + this.#sendMessage(message, requestId); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } this.#procedureCallbacks.set(requestId, result => { if (result.tag === 'Ok') { resolve(result.value); diff --git a/crates/bindings-typescript/src/sdk/event_emitter.ts b/crates/bindings-typescript/src/sdk/event_emitter.ts index a3447ce4b58..f823e9862d0 100644 --- a/crates/bindings-typescript/src/sdk/event_emitter.ts +++ b/crates/bindings-typescript/src/sdk/event_emitter.ts @@ -19,6 +19,12 @@ export class EventEmitter { callbacks.delete(callback); } + /** @internal Clear callbacks when an explicitly managed generation ends. */ + clear(): void { + for (const callbacks of this.#events.values()) callbacks.clear(); + this.#events.clear(); + } + emit(event: Key, ...args: any[]): void { const callbacks = this.#events.get(event); if (!callbacks) { diff --git a/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts b/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts new file mode 100644 index 00000000000..e41781120db --- /dev/null +++ b/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts @@ -0,0 +1,23 @@ +/** Internal opt-in boundary used only by managed container sessions. */ +export const INTERNAL_MANAGED_SESSION: unique symbol = + Symbol('managed session'); +export const INTERNAL_CLEAR_TABLE_CALLBACKS: unique symbol = Symbol( + 'clear table callbacks' +); + +/** A local terminal outcome, never a claim that the server rolled a call back. */ +export class ContainerSessionCallError extends Error { + constructor(readonly outcome: 'not_sent' | 'unknown') { + super( + outcome === 'not_sent' + ? 'Container session ended before this call was sent' + : 'Container session ended without a confirmed call result; outcome unknown' + ); + this.name = 'ContainerSessionCallError'; + } +} + +export interface ManagedSessionLifecycle { + enable(): void; + seal(error: Error): void; +} diff --git a/crates/bindings-typescript/src/sdk/node/container.ts b/crates/bindings-typescript/src/sdk/node/container.ts new file mode 100644 index 00000000000..77d059f08b0 --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/container.ts @@ -0,0 +1,407 @@ +import { request, type ClientRequest, type IncomingMessage } from 'node:http'; +import { isIP, type Socket } from 'node:net'; +import { performance } from 'node:perf_hooks'; +import { inspect } from 'node:util'; +import { Identity } from '../../lib/identity'; + +const REQUEST_MS = 5_000; +const MAX_BODY = 8192 + 256; +const MAX_HEADERS = 4096; +const LOCAL_AUTHORITY = '127.0.0.1:18081'; +const CREDENTIAL_PATH = '/v1/credentials'; + +export type ContainerCredentialErrorCode = + | 'missing_environment' + | 'invalid_discovery' + | 'invalid_target' + | 'denied' + | 'unavailable' + | 'transport' + | 'timeout' + | 'aborted' + | 'invalid_response'; + +/** Errors contain no discovery values, response bodies or credentials. */ +export class ContainerCredentialError extends Error { + constructor(readonly code: ContainerCredentialErrorCode) { + super(`Container credential request failed: ${code}`); + this.name = 'ContainerCredentialError'; + } +} + +type Endpoint = + | { socketPath: string } + | { hostname: string; port: number; host: string }; + +/** Discovery values injected by the container platform; these are not secrets. */ +export interface ContainerDiscovery { + databaseIdentity: string; + serverUri: string; + credentialBroker: string; +} + +/** + * Explicit discovery and short-lived credentials for a Node.js process. + * No stored CLI credentials, default server, anonymous Identity, redirects or + * proxy configuration are used. Construction performs no I/O. + */ +export class Container { + #identity: string; + #server: string; + #endpoint: Endpoint; + + constructor(discovery: ContainerDiscovery) { + this.#identity = identity(discovery.databaseIdentity, 'invalid_discovery'); + const server = parseUrl(discovery.serverUri); + if (!['http:', 'https:', 'ws:', 'wss:'].includes(server.protocol)) { + throw new ContainerCredentialError('invalid_discovery'); + } + if (!server.hostname) { + throw new ContainerCredentialError('invalid_discovery'); + } + this.#server = server.href; + this.#endpoint = endpoint(discovery.credentialBroker); + } + + /** Read only the three platform discovery variables. */ + static fromEnvironment( + environment: Readonly> = process.env + ): Container { + const read = (name: string): string => { + const value = environment[name]; + if (value === undefined) { + throw new ContainerCredentialError('missing_environment'); + } + return value; + }; + return new Container({ + databaseIdentity: read('SPACETIMEDB_DATABASE_IDENTITY'), + serverUri: read('SPACETIMEDB_SERVER_URI'), + credentialBroker: read('SPACETIMEDB_CREDENTIAL_BROKER'), + }); + } + + get databaseIdentity(): Identity { + return new Identity(this.#identity); + } + + get serverUri(): string { + return this.#server; + } + + /** + * Request a fresh token for an exact target Identity, defaulting to this + * container's database. The broker authenticates the process independently + * of these discovery values. Send the token only to the target's trusted + * server. This does not reconnect a database connection or replay calls. + */ + async tokenFor( + target: Identity = this.databaseIdentity, + signal?: AbortSignal + ): Promise { + let targetHex: string; + try { + targetHex = identity(target.toHexString(), 'invalid_target'); + } catch { + throw new ContainerCredentialError('invalid_target'); + } + const response = await readToken(this.#endpoint, targetHex, signal); + return ContainerToken.fromResponse(response, targetHex); + } + + [inspect.custom](): string { + return 'Container { discovery: }'; + } +} + +/** An in-memory bearer token. Inspection and JSON serialization are redacted. */ +export class ContainerToken { + #token: string; + #target: string; + #expiry: number; + #deadline: number; + + private constructor( + token: string, + target: string, + expiry: number, + remaining: number + ) { + this.#token = token; + this.#target = target; + this.#expiry = expiry; + this.#deadline = performance.now() + remaining; + } + + /** @internal */ + static fromResponse(bytes: Buffer, target: string): ContainerToken { + try { + const value: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes) + ); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error(); + const fields = value as Record; + const token = fields.token; + const seconds = fields.expires_unix_seconds; + if ( + Object.keys(fields).length !== 2 || + typeof token !== 'string' || + !/^[\x21-\x7e]{1,8192}$/.test(token) || + typeof seconds !== 'number' || + !Number.isSafeInteger(seconds) || + !Number.isSafeInteger(seconds * 1000) + ) + throw new Error(); + const expiry = seconds * 1000; + const remaining = expiry - Date.now(); + if (remaining <= 0 || remaining > 30_000) throw new Error(); + return new ContainerToken(token, target, expiry, remaining); + } catch { + throw new ContainerCredentialError('invalid_response'); + } + } + + /** Do not log, persist, or place this value in a URL or environment variable. */ + get value(): string { + return this.#token; + } + + get target(): Identity { + return new Identity(this.#target); + } + + get expiresAt(): Date { + return new Date(this.#expiry); + } + + /** A monotonic cap prevents a backward wall-clock jump from extending it. */ + get remainingLifetimeMs(): number { + return Math.max( + 0, + Math.min(this.#deadline - performance.now(), this.#expiry - Date.now()) + ); + } + + toJSON(): object { + return { + target: this.#target, + expiresAt: this.expiresAt, + token: '', + }; + } + + [inspect.custom](): object { + return this.toJSON(); + } +} + +function identity(value: string, code: ContainerCredentialErrorCode): string { + if (typeof value !== 'string' || !/^[0-9a-fA-F]{64}$/.test(value)) { + throw new ContainerCredentialError(code); + } + return value.toLowerCase(); +} + +function parseUrl(value: string): URL { + try { + if ( + !value || + Buffer.byteLength(value) > 4096 || + [...value].some(c => c.charCodeAt(0) <= 32 || c.charCodeAt(0) === 127) + ) + throw new Error(); + const url = new URL(value); + if ( + url.username || + url.password || + value.includes('?') || + value.includes('#') + ) + throw new Error(); + return url; + } catch { + throw new ContainerCredentialError('invalid_discovery'); + } +} + +function endpoint(value: string): Endpoint { + const url = parseUrl(value); + if (url.protocol === 'unix:') { + const path = value.slice('unix://'.length); + if ( + !value.startsWith('unix:///') || + url.host || + !path.startsWith('/') || + Buffer.byteLength(path) > 103 || + path.includes('%') || + path.includes('\\') || + path.split('/').some(part => part === '.' || part === '..') + ) + throw new ContainerCredentialError('invalid_discovery'); + return { socketPath: path }; + } + // Validate the original authority before URL normalization can turn a DNS + // spelling or an abbreviated/hexadecimal IPv4 address into loopback. + const match = + /^http:\/\/(\[[0-9a-fA-F:]+\]|[0-9.]+)(?::([0-9]+))?\/v1\/credentials$/.exec( + value + ); + if (!match) throw new ContainerCredentialError('invalid_discovery'); + const hostname = match[1].replace(/^\[|\]$/g, ''); + const version = isIP(hostname); + if ( + (version !== 4 || hostname.split('.')[0] !== '127') && + (version !== 6 || new URL(`http://[${hostname}]/`).hostname !== '[::1]') + ) + throw new ContainerCredentialError('invalid_discovery'); + const port = match[2] === undefined ? 80 : Number(match[2]); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new ContainerCredentialError('invalid_discovery'); + } + return { hostname, port, host: url.host }; +} + +function responseLength(response: IncomingMessage): number { + if (response.statusCode === 401 || response.statusCode === 403) { + throw new ContainerCredentialError('denied'); + } + if (response.statusCode === 503) + throw new ContainerCredentialError('unavailable'); + if (response.statusCode !== 200) + throw new ContainerCredentialError('invalid_response'); + const count = (name: string): number => + response.rawHeaders.filter( + (v, i) => i % 2 === 0 && v.toLowerCase() === name + ).length; + const headers = response.headers; + if ( + count('content-type') !== 1 || + headers['content-type'] !== 'application/json' || + count('cache-control') !== 1 || + headers['cache-control'] !== 'no-store' || + count('content-length') !== 1 || + count('transfer-encoding') || + count('content-encoding') || + typeof headers['content-length'] !== 'string' || + !/^[0-9]+$/.test(headers['content-length']) + ) + throw new ContainerCredentialError('invalid_response'); + const length = Number(headers['content-length']); + if (!Number.isInteger(length) || length <= 0 || length > MAX_BODY) { + throw new ContainerCredentialError('invalid_response'); + } + return length; +} + +/** Own the request and its socket through terminal close, on every outcome. */ +async function readToken( + endpoint: Endpoint, + target: string, + signal?: AbortSignal +): Promise { + if (signal?.aborted) throw new ContainerCredentialError('aborted'); + return new Promise((resolve, reject) => { + const body = JSON.stringify({ target_database: target }); + let req: ClientRequest | undefined; + let socket: Socket | undefined; + let requestClosed = false; + let socketClosed = false; + let outcome: + | { error?: ContainerCredentialError; bytes?: Buffer } + | undefined; + let timer: ReturnType | undefined; + const complete = (): void => { + if (!outcome || !requestClosed || (socket && !socketClosed)) return; + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + if (outcome.error) reject(outcome.error); + else resolve(outcome.bytes!); + }; + const finish = (error?: ContainerCredentialError, bytes?: Buffer): void => { + if (!outcome) outcome = { error, bytes }; + req?.destroy(); + socket?.destroy(); + complete(); + }; + const fail = (code: ContainerCredentialErrorCode): void => + finish(new ContainerCredentialError(code)); + const abort = (): void => fail('aborted'); + try { + req = request({ + ...endpoint, + // A fresh private agent owns one connection; ambient proxy and HTTP + // pool settings cannot redirect this credential request. + agent: false, + method: 'POST', + path: CREDENTIAL_PATH, + maxHeaderSize: MAX_HEADERS, + headers: { + Host: 'socketPath' in endpoint ? LOCAL_AUTHORITY : endpoint.host, + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Connection: 'close', + }, + }); + req.once('socket', owned => { + socket = owned; + owned.once('close', () => { + socketClosed = true; + complete(); + }); + if (outcome) owned.destroy(); + }); + req.once('error', () => fail('transport')); + req.once('close', () => { + requestClosed = true; + if (!outcome) fail('transport'); + complete(); + }); + req.once('response', response => { + response.on('error', () => fail('transport')); + let length: number; + try { + length = responseLength(response); + } catch (error) { + finish( + error instanceof ContainerCredentialError + ? error + : new ContainerCredentialError('invalid_response') + ); + return; + } + const chunks: Buffer[] = []; + let received = 0; + response.on('data', (chunk: Buffer) => { + received += chunk.length; + if (received > length) { + fail('invalid_response'); + return; + } + chunks.push(chunk); + }); + response.once('aborted', () => fail('transport')); + response.once('end', () => { + if (received !== length) fail('invalid_response'); + else finish(undefined, Buffer.concat(chunks, received)); + }); + }); + // Upgrades never become an untracked or credential-bearing connection. + req.once('upgrade', (_response, upgraded) => { + upgraded.destroy(); + fail('invalid_response'); + }); + timer = setTimeout(() => fail('timeout'), REQUEST_MS); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + else req.end(body); + } catch { + if (req) fail('transport'); + else { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + reject(new ContainerCredentialError('transport')); + } + } + }); +} diff --git a/crates/bindings-typescript/src/sdk/node/container_session.ts b/crates/bindings-typescript/src/sdk/node/container_session.ts new file mode 100644 index 00000000000..59d09947cb2 --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/container_session.ts @@ -0,0 +1,621 @@ +import { performance } from 'node:perf_hooks'; +import { Identity } from '../../lib/identity'; +import type { DbConnectionBuilder } from '../db_connection_builder'; +import type { DbConnectionImpl } from '../db_connection_impl'; +import { + INTERNAL_MANAGED_SESSION, + type ManagedSessionLifecycle, +} from '../managed_session_lifecycle'; +import type { WebSocketAdapter } from '../ws'; +import { + Container, + ContainerCredentialError, + type ContainerToken, +} from './container'; +import { openNodeWebSocket, type NodeWebSocketAdapter } from './index'; + +export type ContainerSessionErrorCode = + | 'invalid_configuration' + | 'already_running' + | 'terminated' + | 'denied' + | 'credentials_failed' + | 'sender_mismatch' + | 'callback_failed' + | 'message_processing_failed'; +export class ContainerSessionError extends Error { + constructor(readonly code: ContainerSessionErrorCode) { + super(`Container session failed: ${code}`); + this.name = 'ContainerSessionError'; + } +} +export interface ContainerGeneration { + readonly generation: number; + readonly target: Identity; + readonly expiresAt: Date; +} +export type ContainerSessionEvent = + | { readonly type: 'connected'; readonly generation: number } + | { + readonly type: 'disconnected'; + readonly generation: number; + readonly reason: + | 'renewed' + | 'expired' + | 'connection_closed' + | 'paused' + | 'shutdown' + | 'failed'; + readonly unconfirmedCalls: 'unknown'; + } + | { readonly type: 'retrying_credentials' }; +export interface ContainerSessionOptions> { + readonly container: Container; + readonly target?: Identity; + readonly serverUri?: string; + /** Synchronous setup, called after sender verification for every fresh cache. */ + readonly onConnect: (connection: C, generation: ContainerGeneration) => void; + readonly onEvent?: (event: ContainerSessionEvent) => void; + readonly compression?: 'gzip' | 'brotli' | 'none'; + readonly lightMode?: boolean; + readonly confirmedReads?: boolean; +} +type EndReason = Extract< + ContainerSessionEvent, + { type: 'disconnected' } +>['reason']; + +class Wake { + #listeners = new Set<() => void>(); + notify(): void { + for (const listener of this.#listeners) listener(); + } + async wait(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise(resolve => { + const done = (): void => { + clearTimeout(timer); + this.#listeners.delete(done); + signal.removeEventListener('abort', done); + resolve(); + }; + const timer = setTimeout(done, Math.max(0, Math.min(milliseconds, 250))); + this.#listeners.add(done); + signal.addEventListener('abort', done, { once: true }); + if (signal.aborted) done(); + }); + } +} + +/** Drops all late decompression/message callbacks once the generation is sealed. */ +class ManagedSocket implements WebSocketAdapter { + #sealed = false; + #failure?: () => void; + #closed?: () => void; + constructor( + private adapter: NodeWebSocketAdapter, + failure: () => void, + closed: () => void + ) { + this.#failure = failure; + this.#closed = closed; + } + get protocol(): string { + return this.adapter.protocol; + } + get readyState(): number { + return this.adapter.readyState; + } + send(bytes: Uint8Array): void { + if (this.#sealed) throw new ContainerSessionError('terminated'); + this.adapter.send(bytes); + } + seal(): void { + this.#sealed = true; + } + close(): void { + this.seal(); + this.adapter.close(); + } + async shutdown(): Promise { + this.seal(); + await this.adapter.shutdown(); + this.#failure = undefined; + this.#closed = undefined; + this.adapter.onmessage = () => {}; + this.adapter.onopen = () => {}; + this.adapter.onerror = () => {}; + this.adapter.onclose = () => {}; + } + set onopen(handler: () => void) { + this.adapter.onopen = () => { + if (!this.#sealed) { + try { + handler(); + } catch { + this.#failure?.(); + } + } + }; + } + set onmessage(handler: (message: { data: Uint8Array }) => void) { + this.adapter.onmessage = message => { + if (!this.#sealed) { + try { + handler(message); + } catch { + this.#failure?.(); + } + } + }; + } + set onerror(handler: (event: ErrorEvent) => void) { + this.adapter.onerror = event => { + if (!this.#sealed) { + try { + handler(event); + } catch { + this.#failure?.(); + } + } + }; + } + set onclose(handler: (event: CloseEvent) => void) { + this.adapter.onclose = event => { + this.seal(); + this.#closed?.(); + try { + handler(event); + } catch { + this.#failure?.(); + } + }; + } +} +interface Generation { + number: number; + token: ContainerToken; + connection?: C; + lifecycle?: ManagedSessionLifecycle; + socket?: ManagedSocket; + socketPromise?: Promise; + wake: Wake; + connected: boolean; + closed: boolean; + sealed: boolean; + failure?: ContainerSessionError; +} + +/** + * An explicit owner of sequential Node database connections and credential renewal. + * Pass the unmodified generated DbConnection class. Each generation gets its own + * builder, callbacks, subscriptions and cache. run(signal) pauses by closing and + * joining the current request/socket before resolving; a later run resumes with + * fresh credentials. shutdown() is terminal. Always await one of these owners. + * No reducer or procedure is replayed, and closure does not imply rollback. + */ +export class ContainerSession> { + #builder: () => DbConnectionBuilder; + #options: ContainerSessionOptions; + #target: string; + #sender: string; + #server: string; + #running?: Promise; + #abort?: AbortController; + #terminal = false; + #nextGeneration = 0; + #active?: Generation; + #builders = new WeakSet>(); + #wake = new Wake(); + + constructor( + connectionType: { builder(): DbConnectionBuilder }, + options: ContainerSessionOptions + ) { + try { + this.#builder = connectionType.builder.bind(connectionType); + this.#options = { ...options }; + this.#target = (options.target ?? options.container.databaseIdentity) + .toHexString() + .toLowerCase(); + this.#sender = options.container.databaseIdentity + .toHexString() + .toLowerCase(); + if ( + !/^[0-9a-f]{64}$/.test(this.#target) || + !/^[0-9a-f]{64}$/.test(this.#sender) || + typeof options.onConnect !== 'function' + ) + throw new Error(); + const value = options.serverUri ?? options.container.serverUri; + if ( + !value || + Buffer.byteLength(value) > 4096 || + [...value].some( + character => + character.charCodeAt(0) <= 32 || + character.charCodeAt(0) === 127 || + character === '?' || + character === '#' + ) + ) + throw new Error(); + const url = new URL(value); + if ( + !['http:', 'https:', 'ws:', 'wss:'].includes(url.protocol) || + !url.hostname || + url.username || + url.password + ) + throw new Error(); + url.protocol = ['https:', 'wss:'].includes(url.protocol) ? 'wss:' : 'ws:'; + this.#server = url.href; + } catch { + throw new ContainerSessionError('invalid_configuration'); + } + } + + run(signal?: AbortSignal): Promise { + if (this.#terminal) + return Promise.reject(new ContainerSessionError('terminated')); + if (this.#running) + return Promise.reject(new ContainerSessionError('already_running')); + const abort = new AbortController(); + this.#abort = abort; + const pause = (): void => { + if (this.#active) this.#seal(this.#active); + abort.abort(); + }; + signal?.addEventListener('abort', pause, { once: true }); + if (signal?.aborted) pause(); + const running = this.#drive(abort.signal) + .catch(error => { + this.#terminal = true; + throw error instanceof ContainerSessionError + ? error + : new ContainerSessionError('credentials_failed'); + }) + .finally(() => { + signal?.removeEventListener('abort', pause); + this.#abort = undefined; + this.#running = undefined; + }); + this.#running = running; + return running; + } + + async shutdown(): Promise { + this.#terminal = true; + if (this.#active) this.#seal(this.#active); + this.#abort?.abort(); + // run owns and reports its failure. shutdown still positively joins cleanup. + await this.#running?.catch(() => {}); + } + + #invoke(callback: (() => void) | undefined): void { + if (!callback) return; + try { + const result: unknown = callback(); + if ( + result && + typeof (result as PromiseLike).then === 'function' + ) { + // Async application work cannot be cancelled or joined by the SDK. + void Promise.resolve(result).catch(() => {}); + throw new Error(); + } + } catch { + throw new ContainerSessionError('callback_failed'); + } + } + #event(event: ContainerSessionEvent): void { + this.#invoke( + this.#options.onEvent && + (() => this.#options.onEvent!(Object.freeze(event))) + ); + } + #seal(generation: Generation): void { + if (generation.sealed) return; + generation.sealed = true; + generation.socket?.seal(); + try { + generation.lifecycle?.seal(new ContainerSessionError('terminated')); + } catch { + generation.failure ??= new ContainerSessionError('callback_failed'); + } + generation.wake.notify(); + } + async #close(generation: Generation, reason: EndReason): Promise { + this.#seal(generation); + generation.connection?.disconnect(); + const socket = + generation.socket ?? + (await generation.socketPromise?.catch(() => undefined)); + await socket?.shutdown(); + if (this.#active === generation) this.#active = undefined; + this.#event({ + type: 'disconnected', + generation: generation.number, + reason, + unconfirmedCalls: 'unknown', + }); + } + + async #open( + token: ContainerToken, + signal: AbortSignal + ): Promise> { + const generation: Generation = { + number: ++this.#nextGeneration, + token, + wake: new Wake(), + connected: false, + closed: false, + sealed: false, + }; + this.#active = generation; + const fail = (code: ContainerSessionErrorCode): void => { + generation.failure ??= new ContainerSessionError(code); + this.#seal(generation); + }; + try { + const builder = this.#builder(); + if (this.#builders.has(builder)) + throw new ContainerSessionError('invalid_configuration'); + this.#builders.add(builder); + const connection = builder + .withUri(this.#server) + .withDatabaseName(this.#target) + .withToken(token.value) + .withCompression(this.#options.compression ?? 'gzip') + .withLightMode(this.#options.lightMode ?? false) + .withWSFn(args => { + const server = new URL(args.url); + server.searchParams.delete('connection_id'); + if ( + server.href !== this.#server || + args.nameOrAddress !== this.#target || + args.authToken !== token.value + ) + throw new ContainerSessionError('invalid_configuration'); + const promise = openNodeWebSocket(args).then(adapter => { + const socket = new ManagedSocket( + adapter, + () => fail('message_processing_failed'), + () => { + generation.closed = true; + generation.wake.notify(); + } + ); + generation.socket = socket; + if (generation.sealed || signal.aborted) socket.close(); + return socket; + }); + generation.socketPromise = promise; + return promise; + }) + .onConnect((connected, identity) => { + if ( + generation.sealed || + signal.aborted || + token.remainingLifetimeMs <= 0 + ) { + this.#seal(generation); + return; + } + if (identity.toHexString().toLowerCase() !== this.#sender) { + fail('sender_mismatch'); + return; + } + if (generation.connected) { + fail('message_processing_failed'); + return; + } + generation.connected = true; + try { + this.#invoke(() => + this.#options.onConnect( + connected, + Object.freeze({ + generation: generation.number, + target: new Identity(this.#target), + expiresAt: token.expiresAt, + }) + ) + ); + if (!generation.sealed) + this.#event({ type: 'connected', generation: generation.number }); + } catch { + fail('callback_failed'); + } + generation.wake.notify(); + }) + .onConnectError(() => { + generation.closed = true; + generation.wake.notify(); + }) + .onDisconnect(() => { + generation.closed = true; + generation.wake.notify(); + }); + if (this.#options.confirmedReads !== undefined) + connection.withConfirmedReads(this.#options.confirmedReads); + generation.connection = connection.build(); + generation.lifecycle = generation.connection[INTERNAL_MANAGED_SESSION](); + generation.lifecycle.enable(); + return generation; + } catch (error) { + await this.#close(generation, 'failed'); + throw error instanceof ContainerSessionError + ? error + : new ContainerSessionError('invalid_configuration'); + } + } + + async #request( + signal: AbortSignal, + generation?: Generation + ): Promise { + if (signal.aborted) return undefined; + const abort = new AbortController(); + const cancel = (): void => abort.abort(); + signal.addEventListener('abort', cancel, { once: true }); + const wake = generation?.wake ?? this.#wake; + let done = false; + let value: ContainerToken | undefined; + let failure: unknown; + const pending = this.#options.container + .tokenFor(new Identity(this.#target), abort.signal) + .then( + token => { + value = token; + }, + error => { + failure = error; + } + ) + .finally(() => { + done = true; + wake.notify(); + }); + try { + while (!done) { + if ( + signal.aborted || + generation?.closed || + generation?.failure || + (generation && generation.token.remainingLifetimeMs <= 0) + ) { + if (generation) this.#seal(generation); + abort.abort(); + await pending; + return undefined; + } + await wake.wait(250, signal); + } + if (signal.aborted) return undefined; + if (failure) throw failure; + if (!value || value.target.toHexString() !== this.#target) + throw new ContainerSessionError('credentials_failed'); + return value; + } finally { + signal.removeEventListener('abort', cancel); + abort.abort(); + await pending; + } + } + + #transient(error: unknown): boolean { + return ( + error instanceof ContainerCredentialError && + ['transport', 'timeout', 'unavailable'].includes(error.code) + ); + } + #credentialFailure(error: unknown): ContainerSessionError { + return error instanceof ContainerCredentialError && error.code === 'denied' + ? new ContainerSessionError('denied') + : new ContainerSessionError('credentials_failed'); + } + async #delay( + milliseconds: number, + signal: AbortSignal, + generation?: Generation + ): Promise { + const until = performance.now() + milliseconds; + while ( + !signal.aborted && + performance.now() < until && + !generation?.closed && + !generation?.failure && + !(generation && generation.token.remainingLifetimeMs <= 0) + ) { + await (generation?.wake ?? this.#wake).wait( + until - performance.now(), + signal + ); + } + } + async #drive(signal: AbortSignal): Promise { + let next: ContainerToken | undefined; + while (!signal.aborted) { + if (!next || next.remainingLifetimeMs <= 0) { + try { + next = await this.#request(signal); + } catch (error) { + if (!this.#transient(error)) throw this.#credentialFailure(error); + this.#event({ type: 'retrying_credentials' }); + await this.#delay(500, signal); + continue; + } + } + if (!next || signal.aborted) break; + const generation = await this.#open(next, signal); + next = undefined; + let reason: EndReason = 'connection_closed'; + try { + const openingDeadline = performance.now() + 5000; + while ( + !generation.connected && + !generation.closed && + !generation.failure && + !signal.aborted && + generation.token.remainingLifetimeMs > 0 && + performance.now() < openingDeadline + ) + await generation.wake.wait(250, signal); + if (generation.failure) throw generation.failure; + if (generation.connected) { + let wait = Math.max( + 0, + generation.token.remainingLifetimeMs - + Math.min(5000, generation.token.remainingLifetimeMs / 3) + ); + while ( + !signal.aborted && + !generation.closed && + !generation.failure && + generation.token.remainingLifetimeMs > 0 + ) { + await this.#delay(wait, signal, generation); + if ( + signal.aborted || + generation.closed || + generation.failure || + generation.token.remainingLifetimeMs <= 0 + ) + break; + let refreshed: ContainerToken | undefined; + try { + refreshed = await this.#request(signal, generation); + } catch (error) { + if (!this.#transient(error)) throw this.#credentialFailure(error); + this.#event({ type: 'retrying_credentials' }); + } + if ( + refreshed && + refreshed.expiresAt.getTime() >= + generation.token.expiresAt.getTime() + 1000 && + refreshed.remainingLifetimeMs >= + generation.token.remainingLifetimeMs + 1000 + ) { + next = refreshed; + reason = 'renewed'; + break; + } + wait = 500; + } + } + if (generation.failure) throw generation.failure; + if (signal.aborted) reason = this.#terminal ? 'shutdown' : 'paused'; + else if (generation.token.remainingLifetimeMs <= 0) reason = 'expired'; + } catch (error) { + reason = 'failed'; + throw error; + } finally { + await this.#close(generation, reason); + } + if (generation.failure) throw generation.failure; + if (!next && !signal.aborted) await this.#delay(250, signal); + } + } +} diff --git a/crates/bindings-typescript/src/sdk/node/index.ts b/crates/bindings-typescript/src/sdk/node/index.ts new file mode 100644 index 00000000000..7a7390b2bbe --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/index.ts @@ -0,0 +1,166 @@ +import { WebSocket } from 'ws'; +import { WebsocketDecompressAdapter } from '../websocket_decompress_adapter'; +import type { WebSocketArgs } from '../ws'; + +const IO_TIMEOUT_MS = 5_000; + +/** + * A Node.js socket with an awaited transport shutdown. + * Closure does not establish whether an unconfirmed reducer or procedure + * committed. Callers must never automatically replay those requests. + */ +export class NodeWebSocketAdapter extends WebsocketDecompressAdapter { + #socket: WebSocket; + #timer?: ReturnType; + #completion: Promise; + #closing = false; + + constructor(socket: WebSocket) { + super(socket as unknown as globalThis.WebSocket); + this.#socket = socket; + this.#completion = + socket.readyState === WebSocket.CLOSED + ? Promise.resolve() + : new Promise(resolve => { + socket.once('close', () => { + clearTimeout(this.#timer); + resolve(); + }); + }); + // An early failed handshake must be handled even before the SDK installs + // its error callback. The callback below receives only a fixed diagnostic. + socket.on('error', () => {}); + } + + static override async openWebSocket( + args: WebSocketArgs + ): Promise { + return openNodeWebSocket(args); + } + + override set onerror(handler: (event: globalThis.ErrorEvent) => void) { + this.#socket.onerror = () => + handler( + Object.assign(new Event('error'), { + message: 'Database WebSocket transport failed', + filename: '', + lineno: 0, + colno: 0, + error: undefined, + }) + ); + } + + override close(): void { + if (this.#closing || this.#socket.readyState === WebSocket.CLOSED) return; + this.#closing = true; + if (this.#socket.readyState === WebSocket.CONNECTING) { + this.#socket.terminate(); + return; + } + // terminate() owns the upgraded socket, unlike destroying an HTTP pool + // that has already transferred socket ownership to WebSocket. + this.#timer = setTimeout(() => this.#socket.terminate(), IO_TIMEOUT_MS); + this.#socket.close(); + } + + /** Close the WebSocket and await its actual terminal close event. */ + async shutdown(): Promise { + this.close(); + await this.#completion; + } +} + +/** + * Node.js WebSocket factory for `DbConnection.builder().withWSFn(...)`. + * Sends the supplied token in Authorization without exchanging it for a + * generic WebSocket token or placing it in the URL. Install the optional + * `ws` peer dependency before importing `spacetimedb/sdk/node`. + * + * This transports a token supplied by the caller. It does not fetch or renew + * container credentials. An expired connection must obtain fresh credentials + * and create a new database connection and subscriptions. + */ +export async function openNodeWebSocket( + args: WebSocketArgs +): Promise { + const { url, nameOrAddress, authToken } = args; + const query = [...url.searchParams]; + const connectionId = url.searchParams.get('connection_id'); + if ( + !['ws:', 'wss:'].includes(url.protocol) || + url.username || + url.password || + (query.length !== 0 && + (query.length !== 1 || + query[0][0] !== 'connection_id' || + !/^[0-9a-f]{32}$/.test(connectionId ?? ''))) || + url.hash || + !nameOrAddress || + nameOrAddress === '.' || + nameOrAddress === '..' || + nameOrAddress.length > 256 || + [...nameOrAddress].some(character => { + const code = character.charCodeAt(0); + return code < 32 || code === 127; + }) || + (authToken !== undefined && + (!authToken || + authToken.length > 8192 || + !/^[\x21-\x7e]+$/.test(authToken))) + ) { + throw new Error('Invalid Node.js database WebSocket configuration'); + } + const target = new URL( + `v1/database/${encodeURIComponent(nameOrAddress)}/subscribe`, + url + ); + if (connectionId !== null) { + target.searchParams.set('connection_id', connectionId); + } + target.searchParams.set( + 'compression', + { gzip: 'Gzip', brotli: 'Brotli', none: 'None' }[args.compression] + ); + if (args.lightMode) target.searchParams.set('light', 'true'); + if (args.confirmedReads !== undefined) { + target.searchParams.set('confirmed', String(args.confirmedReads)); + } + try { + const socket = new WebSocket(target, args.wsProtocol, { + // A private one-use HTTP agent avoids inherited global proxy settings. + // The WebSocket itself retains its upgraded socket for terminate(). + agent: false, + followRedirects: false, + // Never inherit NODE_TLS_REJECT_UNAUTHORIZED=0 for bearer credentials. + rejectUnauthorized: true, + handshakeTimeout: IO_TIMEOUT_MS, + perMessageDeflate: false, + headers: + authToken === undefined ? {} : { Authorization: `Bearer ${authToken}` }, + }); + socket.binaryType = 'arraybuffer'; + return new NodeWebSocketAdapter(socket); + } catch { + throw new Error('Database WebSocket transport could not start'); + } +} + +export { + Container, + ContainerToken, + ContainerCredentialError, +} from './container'; +export type { + ContainerDiscovery, + ContainerCredentialErrorCode, +} from './container'; + +export { ContainerSession, ContainerSessionError } from './container_session'; +export type { + ContainerGeneration, + ContainerSessionOptions, + ContainerSessionEvent, + ContainerSessionErrorCode, +} from './container_session'; +export { ContainerSessionCallError } from '../managed_session_lifecycle'; diff --git a/crates/bindings-typescript/src/sdk/table_cache.ts b/crates/bindings-typescript/src/sdk/table_cache.ts index 543d3fb78ed..6b68f2c710c 100644 --- a/crates/bindings-typescript/src/sdk/table_cache.ts +++ b/crates/bindings-typescript/src/sdk/table_cache.ts @@ -1,3 +1,4 @@ +import { INTERNAL_CLEAR_TABLE_CALLBACKS } from './managed_session_lifecycle'; import { EventEmitter } from './event_emitter.ts'; import { stdbLogger } from './logger.ts'; @@ -219,6 +220,11 @@ export class TableCacheImpl< } } + /** @internal Release callbacks when an explicitly managed generation ends. */ + [INTERNAL_CLEAR_TABLE_CALLBACKS](): void { + this.emitter.clear(); + } + /** * @returns number of rows in the table */ diff --git a/crates/bindings-typescript/tests/node_container_session.test.ts b/crates/bindings-typescript/tests/node_container_session.test.ts new file mode 100644 index 00000000000..946fe7855d9 --- /dev/null +++ b/crates/bindings-typescript/tests/node_container_session.test.ts @@ -0,0 +1,664 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { createServer, type Server, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import type { AddressInfo, Socket } from 'node:net'; +import { WebSocketServer, WebSocket } from 'ws'; +import { + BinaryReader, + BinaryWriter, + ConnectionId, + Identity, + Timestamp, +} from '../src'; +import { + Container, + ContainerSession, + ContainerSessionError, + ContainerSessionCallError, +} from '../src/sdk/node'; +import { ClientMessage, ServerMessage } from '../src/sdk/client_api/types'; +import { INTERNAL_MANAGED_SESSION } from '../src/sdk/managed_session_lifecycle'; +import WebsocketTestAdapter from '../src/sdk/websocket_test_adapter'; +import { V2_WS_PROTOCOL, V3_WS_PROTOCOL } from '../src/sdk/websocket_protocols'; +import { + decodeClientMessagesV3, + encodeServerMessagesV3, +} from '../src/sdk/websocket_v3_frames'; +import { DbConnection } from '../test-app/src/module_bindings'; +import { encodePlayer, makeQueryRows, makeQuerySetUpdate } from './utils'; + +const OWN = '1'.repeat(64); +const TARGET = '2'.repeat(64); +const sessions: ContainerSession[] = []; +const fixtures: Fixture[] = []; +class Deferred { + resolve!: (value: T) => void; + reject!: (reason?: unknown) => void; + promise = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); +} +function send(socket: WebSocket, message: ServerMessage): void { + const writer = new BinaryWriter(1024); + ServerMessage.serialize(writer, message); + const payload = + socket.protocol === V3_WS_PROTOCOL + ? encodeServerMessagesV3(new BinaryWriter(1024), [writer.getBuffer()]) + : writer.getBuffer(); + socket.send(Buffer.concat([Buffer.from([0]), payload])); +} +function credential( + response: ServerResponse, + expiry: number, + token = 'fixture-session-token' +): void { + const body = JSON.stringify({ token, expires_unix_seconds: expiry }); + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': Buffer.byteLength(body), + }) + .end(body); +} +class Fixture { + server: Server; + ws = new WebSocketServer({ + noServer: true, + handleProtocols: () => this.protocol, + }); + protocol: string = V2_WS_PROTOCOL; + sockets = new Set(); + peerClosures: Promise[] = []; + live = 0; + maxLive = 0; + connections = 0; + requests = 0; + calls: { generation: number; message: ClientMessage }[] = []; + routes: string[] = []; + targets: string[] = []; + serverUri = ''; + broker = ''; + source!: Container; + onRequest: (response: ServerResponse, count: number) => void = response => + credential(response, Math.floor(Date.now() / 1000) + 15); + onConnection?: (socket: WebSocket, generation: number) => void; + onMessage?: ( + socket: WebSocket, + message: ClientMessage, + generation: number + ) => void; + sender = OWN; + initial = true; + constructor() { + this.server = createServer((request, response) => { + if (request.url !== '/v1/credentials') { + response.writeHead(500).end(); + return; + } + let body = ''; + request.on('data', data => { + body += data; + }); + request.on('end', () => { + this.targets.push(JSON.parse(body).target_database); + this.onRequest(response, ++this.requests); + }); + }); + this.server.on('connection', socket => { + this.sockets.add(socket); + socket.once('close', () => this.sockets.delete(socket)); + }); + this.server.on('upgrade', (request, socket, head) => { + this.routes.push(request.url!); + this.ws.handleUpgrade(request, socket, head, client => { + const generation = ++this.connections; + this.live++; + this.maxLive = Math.max(this.live, this.maxLive); + this.peerClosures.push( + new Promise(resolve => + client.once('close', () => { + this.live--; + resolve(); + }) + ) + ); + client.on('error', () => {}); + client.on('message', bytes => { + const data = new Uint8Array(bytes as Buffer); + const messages = + this.protocol === V3_WS_PROTOCOL + ? decodeClientMessagesV3(data) + : [data]; + for (const bytes of messages) { + const message = ClientMessage.deserialize(new BinaryReader(bytes)); + this.calls.push({ generation, message }); + this.onMessage?.(client, message, generation); + if (message.tag === 'Subscribe') { + send( + client, + ServerMessage.SubscribeApplied({ + requestId: message.value.requestId, + querySetId: message.value.querySetId, + rows: makeQueryRows( + 'player', + encodePlayer({ + id: generation, + userId: new Identity(OWN), + name: `generation-${generation}`, + location: { x: 0, y: 0 }, + }) + ), + }) + ); + } + } + }); + if (this.initial) { + const id = new URL(request.url!, 'http://127.0.0.1').searchParams.get( + 'connection_id' + )!; + send( + client, + ServerMessage.InitialConnection({ + identity: new Identity(this.sender), + token: '', + connectionId: new ConnectionId(BigInt('0x' + id)), + }) + ); + } + this.onConnection?.(client, generation); + }); + }); + } + async start(): Promise { + this.server.listen(0, '127.0.0.1'); + await once(this.server, 'listening'); + this.serverUri = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}/`; + this.broker = this.serverUri + 'v1/credentials'; + this.source = new Container({ + databaseIdentity: OWN, + serverUri: this.serverUri, + credentialBroker: this.broker, + }); + fixtures.push(this); + return this; + } + async joinPeers(): Promise { + await Promise.all(this.peerClosures); + } + async close(): Promise { + for (const socket of this.sockets) socket.destroy(); + await new Promise(resolve => this.ws.close(() => resolve())); + await new Promise((resolve, reject) => + this.server.close(error => (error ? reject(error) : resolve())) + ); + } +} +afterEach(async () => { + await Promise.all(sessions.splice(0).map(session => session.shutdown())); + await Promise.all(fixtures.splice(0).map(fixture => fixture.close())); +}); +function own( + fixture: Fixture, + onConnect: (connection: DbConnection, info: { generation: number }) => void, + onEvent?: ConstructorParameters< + typeof ContainerSession + >[1]['onEvent'] +): ContainerSession { + const session = new ContainerSession(DbConnection, { + container: fixture.source, + target: new Identity(TARGET), + onConnect, + onEvent, + compression: 'none', + }); + sessions.push(session); + return session; +} + +describe('managed Node container sessions', () => { + it('renews with a fresh cache/subscription and no overlapping sockets or replay', async () => { + const fixture = await new Fixture().start(); + fixture.onRequest = (response, count) => + credential( + response, + Math.floor(Date.now() / 1000) + (count === 1 ? 3 : 15) + ); + const renewed = new Deferred(); + const connections: DbConnection[] = []; + const rows: number[][] = []; + let uncertain: Promise | undefined; + const session = own(fixture, (connection, info) => { + connections.push(connection); + expect(connection.db.player.count()).toBe(0n); + connection + .subscriptionBuilder() + .onApplied(() => { + rows.push([...connection.db.player.iter()].map(row => row.id)); + if (info.generation === 2) renewed.resolve(); + }) + .subscribe('SELECT * FROM player'); + if (info.generation === 1) + uncertain = connection.reducers + .createPlayer({ name: 'once', location: { x: 0, y: 0 } }) + .catch(error => error); + }); + const run = session.run(); + await renewed.promise; + expect(await uncertain).toMatchObject({ outcome: 'unknown' }); + expect(rows).toEqual([[1], [2]]); + expect(fixture.maxLive).toBe(1); + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(1); + expect(fixture.targets.every(target => target === TARGET)).toBe(true); + expect( + fixture.routes.every(route => + route.startsWith(`/v1/database/${TARGET}/subscribe?`) + ) + ).toBe(true); + await expect( + connections[0].reducers.createPlayer({ + name: 'late', + location: { x: 0, y: 0 }, + }) + ).rejects.toMatchObject({ outcome: 'not_sent' }); + await expect( + connections[0].callProcedure('late', new Uint8Array()) + ).rejects.toMatchObject({ outcome: 'not_sent' }); + expect(() => + connections[0].subscriptionBuilder().subscribe('SELECT * FROM player') + ).toThrow(ContainerSessionError); + await session.shutdown(); + await run; + }, 10000); + + it('does not rotate repeatedly on the same lease expiry and denial is terminal', async () => { + const fixture = await new Fixture().start(); + const expiry = Math.floor(Date.now() / 1000) + 4; + fixture.onRequest = (response, count) => + count < 3 + ? credential(response, expiry) + : response.writeHead(403, { 'Content-Length': 0 }).end(); + let connected = 0; + const session = own(fixture, () => { + connected++; + }); + await expect(session.run()).rejects.toMatchObject({ code: 'denied' }); + expect(fixture.requests).toBe(3); + expect(connected).toBe(1); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + await expect(session.run()).rejects.toMatchObject({ code: 'terminated' }); + }, 10000); + + it('expires and closes the old socket while a refresh response is stalled', async () => { + const fixture = await new Fixture().start(); + fixture.onRequest = (response, count) => { + if (count === 1) credential(response, Math.floor(Date.now() / 1000) + 3); + }; + const expired = new Deferred(); + const pause = new AbortController(); + const session = own( + fixture, + () => {}, + event => { + if (event.type === 'disconnected' && event.reason === 'expired') { + expired.resolve(); + pause.abort(); + } + } + ); + const run = session.run(pause.signal); + await expired.promise; + await run; + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + expect(fixture.connections).toBe(1); + }, 10000); + + it('joins cancellation and can resume with another fresh generation', async () => { + const fixture = await new Fixture().start(); + let pause = new AbortController(); + const session = own(fixture, () => pause.abort()); + await session.run(pause.signal); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + pause = new AbortController(); + await session.run(pause.signal); + expect(fixture.connections).toBe(2); + expect(fixture.requests).toBe(2); + expect(fixture.maxLive).toBe(1); + }); + + it('rejects a wrong sender before exposing onConnect, including immediate close', async () => { + for (const immediateClose of [false, true]) { + const fixture = await new Fixture().start(); + fixture.sender = TARGET; + if (immediateClose) fixture.onConnection = socket => socket.close(); + let exposed = 0; + const session = own(fixture, () => { + exposed++; + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'sender_mismatch', + }); + expect(exposed).toBe(0); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + } + }); + + it('keeps a confirmed result when a later seal wins other pending calls', async () => { + const fixture = await new Fixture().start(); + const result = new Deferred(); + fixture.onMessage = (socket, message) => { + if (message.tag === 'CallReducer') + send( + socket, + ServerMessage.ReducerResult({ + requestId: message.value.requestId, + timestamp: new Timestamp(0n), + result: { tag: 'OkEmpty' }, + }) + ); + }; + const pause = new AbortController(); + const session = own(fixture, connection => { + void connection.reducers + .createPlayer({ name: 'confirmed', location: { x: 0, y: 0 } }) + .then( + () => { + result.resolve(); + pause.abort(); + }, + error => result.reject(error) + ); + }); + const run = session.run(pause.signal); + await result.promise; + await run; + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(1); + }); + + it('rejects a queued managed call as not sent before socket readiness', async () => { + const fixture = await new Fixture().start(); + const ready = new Deferred(); + const session = own(fixture, connection => ready.resolve(connection)); + const run = session.run(); + const connection = await ready.promise; + connection.isActive = false; + const queued = connection.reducers + .createPlayer({ name: 'queued', location: { x: 0, y: 0 } }) + .catch(error => error); + await session.shutdown(); + await run; + expect(await queued).toMatchObject({ outcome: 'not_sent' }); + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(0); + }); + + it('seals before subscription callbacks and joins even when a callback throws', async () => { + const fixture = await new Fixture().start(); + const applied = new Deferred(); + let oldCall: Promise | undefined; + let laterSubscriptionErrors = 0; + const callbackErrors: unknown[] = []; + const session = own(fixture, connection => { + connection + .subscriptionBuilder() + .onApplied(() => applied.resolve()) + .onError(context => { + callbackErrors.push(context.event); + expect(context.isActive).toBe(false); + oldCall = context.reducers + .createPlayer({ name: 'after-seal', location: { x: 0, y: 0 } }) + .catch(error => error); + throw new Error('application-private-error'); + }) + .subscribe('SELECT * FROM player'); + connection + .subscriptionBuilder() + .onError(context => { + callbackErrors.push(context.event); + laterSubscriptionErrors++; + }) + .subscribe('SELECT * FROM player'); + }); + const run = session.run().catch(error => error); + await applied.promise; + await session.shutdown(); + expect(await run).toMatchObject({ code: 'callback_failed' }); + expect(await oldCall).toBeInstanceOf(ContainerSessionCallError); + expect(await oldCall).toMatchObject({ outcome: 'not_sent' }); + expect(laterSubscriptionErrors).toBe(1); + expect(callbackErrors).toHaveLength(2); + expect(callbackErrors[0]).toBeInstanceOf(ContainerSessionError); + expect(callbackErrors[0]).toMatchObject({ code: 'terminated' }); + expect(callbackErrors[1]).toBe(callbackErrors[0]); + await session.shutdown(); + expect(laterSubscriptionErrors).toBe(1); + expect(callbackErrors).toHaveLength(2); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('joins the socket before reporting an application callback failure', async () => { + const fixture = await new Fixture().start(); + const session = own(fixture, () => { + throw new Error('private app failure'); + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'callback_failed', + }); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('cancels an in-flight initial credential request and owns its socket until close', async () => { + const fixture = await new Fixture().start(); + const received = new Deferred(); + let brokerClosed: Promise | undefined; + fixture.onRequest = response => { + brokerClosed = once(response.socket!, 'close'); + received.resolve(); + }; + const pause = new AbortController(); + const session = own(fixture, () => { + throw new Error('must not connect'); + }); + const run = session.run(pause.signal); + await received.promise; + pause.abort(); + await run; + await brokerClosed; + expect(fixture.connections).toBe(0); + expect(fixture.requests).toBe(1); + }); + + it('classifies v3 batched calls as unsent when shutdown seals before the flush', async () => { + const fixture = new Fixture(); + fixture.protocol = V3_WS_PROTOCOL; + await fixture.start(); + const pause = new AbortController(); + let reducer: Promise | undefined; + let procedure: Promise | undefined; + const session = own(fixture, connection => { + reducer = connection.reducers + .createPlayer({ name: 'queued-v3', location: { x: 0, y: 0 } }) + .catch(error => error); + procedure = connection + .callProcedure('queued_procedure', new Uint8Array()) + .catch(error => error); + pause.abort(); + }); + await session.run(pause.signal); + await fixture.joinPeers(); + expect(await reducer).toMatchObject({ outcome: 'not_sent' }); + expect(await procedure).toMatchObject({ outcome: 'not_sent' }); + expect(fixture.calls).toHaveLength(0); + }); + + it('classifies v3 reducer/procedure handoffs without results as unknown', async () => { + const fixture = new Fixture(); + fixture.protocol = V3_WS_PROTOCOL; + await fixture.start(); + const received = new Deferred(); + fixture.onMessage = () => { + if (fixture.calls.length === 2) received.resolve(); + }; + let reducer: Promise | undefined; + let procedure: Promise | undefined; + const session = own(fixture, connection => { + reducer = connection.reducers + .createPlayer({ name: 'sent-v3', location: { x: 0, y: 0 } }) + .catch(error => error); + procedure = connection + .callProcedure('sent_procedure', new Uint8Array()) + .catch(error => error); + }); + const run = session.run(); + await received.promise; + await session.shutdown(); + await run; + await fixture.joinPeers(); + expect(await reducer).toMatchObject({ outcome: 'unknown' }); + expect(await procedure).toMatchObject({ outcome: 'unknown' }); + expect(fixture.calls).toHaveLength(2); + }); + + it('reports unsupported async setup after joining, without an unhandled rejection', async () => { + const fixture = await new Fixture().start(); + const session = own(fixture, async () => { + await Promise.resolve(); + throw new Error('private async callback'); + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'callback_failed', + }); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('keeps a confirmed reducer result when its row callback initiates shutdown', async () => { + const fixture = await new Fixture().start(); + const confirmed = new Deferred(); + let querySetId = 0; + fixture.onMessage = (socket, message) => { + if (message.tag === 'Subscribe') querySetId = message.value.querySetId.id; + if (message.tag === 'CallReducer') + send( + socket, + ServerMessage.ReducerResult({ + requestId: message.value.requestId, + timestamp: new Timestamp(0n), + result: { + tag: 'Ok', + value: { + retValue: new Uint8Array(), + transactionUpdate: { + querySets: [ + makeQuerySetUpdate( + querySetId, + 'player', + encodePlayer({ + id: 99, + userId: new Identity(OWN), + name: 'confirmed', + location: { x: 0, y: 0 }, + }) + ), + ], + }, + }, + }, + }) + ); + }; + const session = own(fixture, connection => { + connection.db.player.onInsert(context => { + if (context.event.tag === 'Reducer') void session.shutdown(); + }); + connection + .subscriptionBuilder() + .onApplied(() => { + void connection.reducers + .createPlayer({ name: 'confirmed', location: { x: 0, y: 0 } }) + .then( + () => confirmed.resolve(), + error => confirmed.reject(error) + ); + }) + .subscribe('SELECT * FROM player'); + }); + const run = session.run(); + await confirmed.promise; + await run; + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('rejects invalid server configuration before broker I/O and owns run state', async () => { + const fixture = await new Fixture().start(); + for (const serverUri of [ + fixture.serverUri + '?token=secret', + fixture.serverUri + '#fragment', + fixture.serverUri + '\u0000', + 'http://user:secret@127.0.0.1/', + ]) { + expect( + () => + new ContainerSession(DbConnection, { + container: fixture.source, + serverUri, + onConnect: () => {}, + }) + ).toThrow(ContainerSessionError); + } + expect(fixture.requests).toBe(0); + const session = own(fixture, () => {}); + const stopped = new AbortController(); + stopped.abort(); + await session.run(stopped.signal); + expect(fixture.requests).toBe(0); + const pause = new AbortController(); + const run = session.run(pause.signal); + await expect(session.run()).rejects.toMatchObject({ + code: 'already_running', + }); + pause.abort(); + await run; + await session.shutdown(); + await expect(session.run()).rejects.toMatchObject({ code: 'terminated' }); + }); + + it('rejects adopting a connection with subscriptions already queued', async () => { + const adapter = new WebsocketTestAdapter(); + const closed = new Deferred(); + const close = adapter.close.bind(adapter); + adapter.close = () => { + close(); + closed.resolve(); + }; + const connection = DbConnection.builder() + .withUri('http://127.0.0.1:1') + .withDatabaseName(TARGET) + .withWSFn(adapter.openWebSocket) + .build(); + try { + connection.subscriptionBuilder().subscribe('SELECT * FROM player'); + expect(() => connection[INTERNAL_MANAGED_SESSION]().enable()).toThrow( + 'Managed boundary requires an unused connection' + ); + } finally { + connection.disconnect(); + await closed.promise; + expect(adapter.closed).toBe(true); + } + }); +}); diff --git a/crates/bindings-typescript/tests/node_credentials.test.ts b/crates/bindings-typescript/tests/node_credentials.test.ts new file mode 100644 index 00000000000..8701e7e348e --- /dev/null +++ b/crates/bindings-typescript/tests/node_credentials.test.ts @@ -0,0 +1,376 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { + createServer, + type Server, + type ServerResponse, + type RequestListener, +} from 'node:http'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { + createServer as createNetServer, + type Server as NetServer, + type AddressInfo, + type Socket, +} from 'node:net'; +import { join } from 'node:path'; +import { inspect } from 'node:util'; +import { Container, ContainerCredentialError } from '../src/sdk/node/container'; +import { Identity } from '../src/lib/identity'; + +const OWN = '1'.repeat(64); +const OTHER = '2'.repeat(64); +const SECRET = 'opaque-fixture-credential'; +const servers: NetServer[] = []; +const sockets = new Set(); +const directories: string[] = []; + +async function fixture( + handler: RequestListener, + unix = false +): Promise<{ server: Server; broker: string }> { + const server = createServer(handler); + servers.push(server); + server.on('connection', socket => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + if (unix) { + const directory = await mkdtemp('/tmp/node-creds-'); + directories.push(directory); + const path = join(directory, 'sock'); + server.listen(path); + await once(server, 'listening'); + return { server, broker: `unix://${path}` }; + } + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + server, + broker: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/credentials`, + }; +} + +async function rawFixture( + reply: string, + truncate = false +): Promise<{ server: NetServer; broker: string }> { + const server = createNetServer(socket => { + sockets.add(socket); + socket.on('error', () => {}); + socket.on('close', () => sockets.delete(socket)); + // A raw upgraded peer can retain its writable half after client EOF. + socket.on('end', () => socket.end()); + socket.once('data', () => { + socket.write(reply); + if (truncate) socket.end(); + }); + }); + servers.push(server); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + server, + broker: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/credentials`, + }; +} + +async function rejectsAndClosesPeer( + reply: string, + code: string, + truncate = false +): Promise { + const { server, broker } = await rawFixture(reply, truncate); + const connected = once(server, 'connection'); + const result = expect(container(broker).tokenFor()).rejects.toMatchObject({ + code, + }); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + await result; + // Observe closure before afterEach can destroy a retained socket. + await closed; + expect(peer.destroyed).toBe(true); +} + +afterEach(async () => { + for (const socket of sockets) socket.destroy(); + await Promise.all( + servers.splice(0).map( + server => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }) + ) + ); + await Promise.all( + directories + .splice(0) + .map(path => rm(path, { recursive: true, force: true })) + ); +}); + +function container(broker: string): Container { + return new Container({ + databaseIdentity: OWN, + serverUri: 'https://127.0.0.1:1/', + credentialBroker: broker, + }); +} + +function success( + response: ServerResponse, + value: unknown = { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 10, + } +): void { + const body = JSON.stringify(value); + response.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': Buffer.byteLength(body), + }); + response.end(body); +} + +describe('Node container credential helper', () => { + it('uses actual loopback HTTP framing, fresh requests and exact target identities', async () => { + const requests: { target: string; headers: object; url?: string }[] = []; + const { broker } = await fixture((request, response) => { + let body = ''; + request.setEncoding('utf8'); + request.on('data', chunk => { + body += chunk; + }); + request.on('end', () => { + requests.push({ + target: body, + headers: request.headers, + url: request.url, + }); + success(response); + }); + }); + const discovered = container(broker); + expect(discovered.databaseIdentity.toHexString()).toBe(OWN); + const mutableCopy = discovered.databaseIdentity; + mutableCopy.__identity__ = 0n; + const first = await discovered.tokenFor(); + const second = await discovered.tokenFor(new Identity(OTHER)); + expect(requests.map(r => r.target)).toEqual([ + JSON.stringify({ target_database: OWN }), + JSON.stringify({ target_database: OTHER }), + ]); + for (const request of requests) { + expect(request.url).toBe('/v1/credentials'); + expect(request.headers).toMatchObject({ + 'content-type': 'application/json', + connection: 'close', + }); + expect(request.headers).not.toHaveProperty('authorization'); + } + expect(first.target.toHexString()).toBe(OWN); + expect(second.target.toHexString()).toBe(OTHER); + expect(first.value).toBe(SECRET); + expect(first.remainingLifetimeMs).toBeGreaterThan(0); + expect(first.remainingLifetimeMs).toBeLessThanOrEqual(10_000); + expect(inspect(first)).not.toContain(SECRET); + expect(JSON.stringify(first)).not.toContain(SECRET); + }); + + it('uses only the owned Unix socket and required logical HTTP authority', async () => { + const hosts: (string | undefined)[] = []; + const { broker } = await fixture((request, response) => { + hosts.push(request.headers.host); + request.resume(); + request.on('end', () => success(response)); + }, true); + const token = await container(broker).tokenFor(); + expect(token.value).toBe(SECRET); + expect(hosts).toEqual(['127.0.0.1:18081']); + }); + + it('rejects invalid discovery without connecting or choosing defaults', () => { + const invalid = [ + 'http://localhost:18081/v1/credentials', + 'http://127.1:18081/v1/credentials', + 'http://0x7f000001:18081/v1/credentials', + 'http://192.0.2.1:18081/v1/credentials', + 'http://user:secret@127.0.0.1:18081/v1/credentials', + 'http://127.0.0.1:18081/v1/credentials?', + 'http://127.0.0.1:18081/v1/credentials#', + 'http://127.0.0.1:0/v1/credentials', + 'https://127.0.0.1:18081/v1/credentials', + 'unix://elsewhere/run/socket', + 'unix:///run/../socket', + 'unix:///run/%73ocket', + `unix:///${'a'.repeat(104)}`, + ]; + for (const broker of invalid) { + expect(() => container(broker)).toThrow(ContainerCredentialError); + } + expect( + () => + new Container({ + databaseIdentity: OWN, + serverUri: `https://127.0.0.1/${'é'.repeat(2100)}`, + credentialBroker: 'unix:///run/credentials.sock', + }) + ).toThrow('invalid_discovery'); + expect(() => Container.fromEnvironment({})).toThrow('missing_environment'); + const discovered = Container.fromEnvironment({ + SPACETIMEDB_DATABASE_IDENTITY: OWN, + SPACETIMEDB_SERVER_URI: 'https://127.0.0.1:1/', + SPACETIMEDB_CREDENTIAL_BROKER: 'unix:///run/spacetimedb/credentials.sock', + }); + expect(discovered.databaseIdentity.toHexString()).toBe(OWN); + expect(inspect(discovered)).not.toContain('/run/spacetimedb'); + }); + + it('does not follow redirects or retry denial, and preserves a later valid request', async () => { + let count = 0; + const { broker } = await fixture((request, response) => { + request.resume(); + const status = [302, 403, 503, 200][count++]; + if (status === 200) success(response); + else + response + .writeHead(status, { + Location: 'http://192.0.2.1/secret', + 'Content-Length': 0, + }) + .end(); + }); + const discovered = container(broker); + for (const code of ['invalid_response', 'denied', 'unavailable']) { + await expect(discovered.tokenFor()).rejects.toMatchObject({ code }); + } + expect((await discovered.tokenFor()).value).toBe(SECRET); + expect(count).toBe(4); + }); + + it('rejects missing no-store, oversized bodies, wrong fields and invalid expiry without exposing tokens', async () => { + const values: unknown[] = [ + { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) - 1, + }, + { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 60, + }, + { token: SECRET, expires_unix_seconds: '123' }, + { token: SECRET, expires_unix_seconds: 1, extra: true }, + { + token: '\nsecret', + expires_unix_seconds: Math.floor(Date.now() / 1000) + 10, + }, + ]; + let count = 0; + const { broker } = await fixture((request, response) => { + request.resume(); + if (count === 0) { + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Content-Length': 2, + }) + .end('{}'); + } else if (count === 1) { + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': 100000, + }) + .end(); + } else success(response, values[count - 2]); + count++; + }); + for (let i = 0; i < values.length + 2; i++) { + const error = await container(broker) + .tokenFor() + .catch(error => error as unknown); + expect(error).toMatchObject({ code: 'invalid_response' }); + expect(inspect(error)).not.toContain(SECRET); + } + }); + + it('aborts a held response and observes peer closure before fixture cleanup', async () => { + const { server, broker } = await fixture(request => { + request.resume(); + }); + const connected = once(server, 'connection'); + const abort = new AbortController(); + const result = container(broker).tokenFor(undefined, abort.signal); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + abort.abort(); + await expect(result).rejects.toMatchObject({ code: 'aborted' }); + await closed; + expect(peer.destroyed).toBe(true); + await expect( + container(broker).tokenFor(undefined, abort.signal) + ).rejects.toMatchObject({ code: 'aborted' }); + }); + + it('bounds a stalled response by five seconds and destroys its actual socket', async () => { + const { server, broker } = await fixture(request => { + request.resume(); + }); + const connected = once(server, 'connection'); + const result = container(broker).tokenFor(); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + await expect(result).rejects.toMatchObject({ code: 'timeout' }); + await closed; + expect(peer.destroyed).toBe(true); + }, 10_000); + + it('rejects an invalid exact target before opening a broker connection', async () => { + let accepted = 0; + const { server, broker } = await fixture(request => request.resume()); + server.on('connection', () => accepted++); + const target = new Identity(OTHER); + target.toHexString = () => 'not-an-identity'; + await expect(container(broker).tokenFor(target)).rejects.toMatchObject({ + code: 'invalid_target', + }); + expect(accepted).toBe(0); + }); + + it('closes a request cancelled before its socket is assigned', async () => { + const { broker } = await fixture(request => request.resume()); + const abort = new AbortController(); + const result = container(broker).tokenFor(undefined, abort.signal); + abort.abort(); + await expect(result).rejects.toMatchObject({ code: 'aborted' }); + }); + + it('rejects duplicate response headers and closes the actual peer', async () => { + const body = JSON.stringify({ + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 20, + }); + await rejectsAndClosesPeer( + `HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`, + 'invalid_response' + ); + }); + + it('rejects a truncated response and closes the actual peer', async () => { + await rejectsAndClosesPeer( + 'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nContent-Length: 100\r\n\r\n{}', + 'transport', + true + ); + }); + + it('rejects an upgrade while retaining ownership through actual socket close', async () => { + await rejectsAndClosesPeer( + 'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: arbitrary\r\n\r\n', + 'invalid_response' + ); + }); +}); diff --git a/crates/bindings-typescript/tests/node_websocket.test.ts b/crates/bindings-typescript/tests/node_websocket.test.ts new file mode 100644 index 00000000000..d585bd056a6 --- /dev/null +++ b/crates/bindings-typescript/tests/node_websocket.test.ts @@ -0,0 +1,324 @@ +import { createHash } from 'node:crypto'; +import { createServer, type IncomingMessage } from 'node:http'; +import type { Socket } from 'node:net'; +import { afterEach, describe, expect, it } from 'vitest'; +import { openNodeWebSocket, NodeWebSocketAdapter } from '../src/sdk/node'; +import { WebSocket } from 'ws'; +import { once } from 'node:events'; +import type { WebSocketArgs } from '../src/sdk/ws'; +import { DbConnection } from '../test-app/src/module_bindings'; +import { BinaryWriter, ConnectionId, Identity } from '../src'; +import { ServerMessage } from '../src/sdk/client_api/types'; + +const sockets: Set = new Set(); +const adapters: NodeWebSocketAdapter[] = []; +const servers: ReturnType[] = []; +const token = 'fixture.hosted.credential'; + +afterEach(async () => { + const closed = await Promise.allSettled( + adapters.splice(0).map(a => a.shutdown()) + ); + for (const socket of sockets) socket.destroy(); + await Promise.all( + servers.splice(0).map( + server => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }) + ) + ); + sockets.clear(); + for (const result of closed) expect(result.status).toBe('fulfilled'); +}); + +async function endpoint( + upgrade: (req: IncomingMessage, socket: Socket) => void +) { + const server = createServer((_req, res) => { + // A token-exchange request must never be sent by this transport. + res.writeHead(500).end(); + }); + server.on('connection', socket => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + server.on('upgrade', (req, socket) => upgrade(req, socket as Socket)); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + servers.push(server); + const address = server.address(); + if (!address || typeof address === 'string') + throw new Error('Expected owned TCP listener'); + return new URL(`ws://127.0.0.1:${address.port}/`); +} + +function args(url: URL): WebSocketArgs { + return { + url, + nameOrAddress: 'test/database', + authToken: token, + wsProtocol: ['fixture-protocol'], + compression: 'none', + lightMode: true, + confirmedReads: false, + }; +} + +function accept( + req: IncomingMessage, + socket: Socket, + payload = Buffer.from([0, 111, 107]), + answerClose = true +) { + const accept = createHash('sha1') + .update( + req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' + ) + .digest('base64'); + socket.write( + 'HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n' + + `Sec-WebSocket-Accept: ${accept}\r\nSec-WebSocket-Protocol: ${req.headers['sec-websocket-protocol']?.split(',')[0].trim()}\r\n\r\n` + ); + // A single uncompressed SpacetimeDB payload inside a binary WS frame. + if (payload.length >= 126) + throw new Error('Fixture message exceeded short-frame bound'); + socket.write(Buffer.concat([Buffer.from([0x82, payload.length]), payload])); + socket.on('data', () => { + if (answerClose) socket.end(Buffer.from([0x88, 0])); + }); +} + +describe('explicit Node.js header transport', () => { + it('connects through the generated builder with its actual connection ID', async () => { + let request: IncomingMessage | undefined; + const identity = new Identity(123n); + const url = await endpoint((req, socket) => { + request = req; + const connectionId = new URL( + req.url!, + 'http://127.0.0.1' + ).searchParams.get('connection_id'); + if (!connectionId) throw new Error('Builder connection ID missing'); + const writer = new BinaryWriter(1024); + ServerMessage.serialize( + writer, + ServerMessage.InitialConnection({ + identity, + token, + connectionId: new ConnectionId(BigInt(`0x${connectionId}`)), + }) + ); + accept( + req, + socket, + Buffer.concat([Buffer.from([0]), writer.getBuffer()]) + ); + }); + let resolve!: () => void; + let reject!: (error: Error) => void; + const ready = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + const connection = DbConnection.builder() + .withUri(url) + .withDatabaseName('fixture') + .withToken(token) + .withWSFn(async input => { + const adapter = await openNodeWebSocket(input); + adapters.push(adapter); + return adapter; + }) + .onConnect((_connection, actualIdentity) => { + expect(actualIdentity.toHexString()).toBe(identity.toHexString()); + resolve(); + }) + .onConnectError((_context, error) => reject(error)) + .build(); + try { + await ready; + expect(request?.headers.authorization).toBe(`Bearer ${token}`); + expect( + new URL(request!.url!, url).searchParams.get('connection_id') + ).toMatch(/^[0-9a-f]{32}$/); + } finally { + connection.disconnect(); + } + }); + it('uses Authorization on the exact route and preserves subscription options', async () => { + let request: IncomingMessage | undefined; + const url = await endpoint((req, socket) => { + request = req; + accept(req, socket); + }); + // The exported Node class's static factory must also use headers directly. + const adapter = await NodeWebSocketAdapter.openWebSocket(args(url)); + adapters.push(adapter); + const received = await new Promise((resolve, reject) => { + adapter.onmessage = event => resolve(event.data); + adapter.onerror = () => reject(new Error('Fixture connection failed')); + }); + expect(new TextDecoder().decode(received)).toBe('ok'); + expect(request?.headers.authorization).toBe(`Bearer ${token}`); + expect(request?.url).toBe( + '/v1/database/test%2Fdatabase/subscribe?compression=None&light=true&confirmed=false' + ); + expect(request?.url).not.toContain(token); + expect(adapter.protocol).toBe('fixture-protocol'); + await Promise.all([adapter.shutdown(), adapter.shutdown()]); + expect(adapter.readyState).toBe(3); + }); + + it('does not follow an upgrade redirect with credentials', async () => { + let forwarded = false; + const destination = await endpoint((req, socket) => { + forwarded = true; + accept(req, socket); + }); + const url = await endpoint((_req, socket) => { + socket.end( + `HTTP/1.1 302 Found\r\nLocation: ${destination}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n` + ); + }); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + const error = await new Promise(resolve => { + adapter.onerror = resolve; + }); + expect(error.message).toBe('Database WebSocket transport failed'); + expect(error.error).toBeUndefined(); + await adapter.shutdown(); + expect(forwarded).toBe(false); + }); + + it('rejects ambiguous URLs and malformed credentials before connecting', async () => { + let connected = false; + const url = await endpoint((req, socket) => { + connected = true; + accept(req, socket); + }); + const invalid = [ + { ...args(url), authToken: 'secret\r\ninjected: header' }, + { ...args(url), authToken: 'x'.repeat(8193) }, + { ...args(url), authToken: '' }, + { ...args(url), nameOrAddress: '.' }, + { ...args(url), nameOrAddress: '..' }, + { ...args(url), nameOrAddress: 'database\n' }, + { ...args(url), nameOrAddress: 'database\u007f' }, + { ...args(url), url: new URL('?connection_id=invalid', url) }, + { + ...args(url), + url: new URL( + '?connection_id=00000000000000000000000000000001&connection_id=00000000000000000000000000000002', + url + ), + }, + { ...args(url), url: new URL('?token=secret', url) }, + { ...args(url), url: new URL('#secret', url) }, + { ...args(url), url: new URL(`ws://secret:secret@${url.host}/`) }, + { ...args(url), url: new URL('file:///not-a-server') }, + ]; + for (const input of invalid) { + await expect(openNodeWebSocket(input)).rejects.toThrow( + 'Invalid Node.js database WebSocket configuration' + ); + } + expect(connected).toBe(false); + }); + + it('immediately completes shutdown when wrapping an already closed socket', async () => { + const url = await endpoint((req, socket) => accept(req, socket)); + const raw = new WebSocket(url, ['fixture-protocol']); + raw.on('error', () => {}); + try { + await once(raw, 'open'); + const closed = once(raw, 'close'); + raw.close(); + await closed; + const adapter = new NodeWebSocketAdapter(raw); + adapters.push(adapter); + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + } finally { + raw.terminate(); + } + }); + + it('cleans up a connection cancelled during its upgrade', async () => { + let admitted!: () => void; + const accepted = new Promise(resolve => { + admitted = resolve; + }); + const url = await endpoint(() => admitted()); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await accepted; + let closeEvents = 0; + adapter.onclose = () => { + closeEvents++; + }; + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + expect(closeEvents).toBe(1); + }); + + it('terminates the actual upgraded socket when the peer ignores close', async () => { + let peerClosed!: () => void; + const closed = new Promise(resolve => { + peerClosed = resolve; + }); + const peerEvents: string[] = []; + const url = await endpoint((req, socket) => { + // Node's upgraded HTTP socket keeps its writable half open after EOF. + // Do not confuse that with a live client transport. Observe the client's + // actual FIN first, then finish the fixture's own writable half. + socket.once('end', () => { + peerEvents.push('end'); + socket.end(); + }); + socket.once('close', () => { + peerEvents.push('close'); + peerClosed(); + }); + accept(req, socket, Buffer.from([0, 111, 107]), false); + }); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await new Promise(resolve => { + adapter.onmessage = () => resolve(); + }); + let closeEvents = 0; + adapter.onclose = () => { + closeEvents++; + }; + await adapter.shutdown(); + await closed; + expect(peerEvents).toEqual(['end', 'close']); + expect(adapter.readyState).toBe(3); + expect(closeEvents).toBe(1); + }, 10_000); + + it('bounds a stalled upgrade without waiting for a caller to close it', async () => { + const url = await endpoint(() => {}); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await new Promise(resolve => { + adapter.onerror = () => resolve(); + }); + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + }, 10_000); + + it('redacts a constructor failure before opening a socket', async () => { + const url = await endpoint(() => {}); + await expect( + openNodeWebSocket({ + ...args(url), + wsProtocol: ['private invalid protocol'], + }) + ).rejects.toThrow('Database WebSocket transport could not start'); + }); +}); diff --git a/crates/bindings-typescript/tsup.config.ts b/crates/bindings-typescript/tsup.config.ts index 80c4af53d31..30f2953004e 100644 --- a/crates/bindings-typescript/tsup.config.ts +++ b/crates/bindings-typescript/tsup.config.ts @@ -220,6 +220,20 @@ export default defineConfig([ esbuildOptions: commonEsbuildTweaks(), }, + // Explicit Node.js transport, excluded from browser entry points. + { + entry: { index: 'src/sdk/node/index.ts' }, + format: ['esm', 'cjs'], + target: 'es2022', + outDir: 'dist/sdk/node', + dts: false, + sourcemap: true, + clean: true, + platform: 'node', + external: ['ws'], + outExtension, + }, + // SDK browser ESM: dist/sdk/index.browser.mjs { entry: { 'index.browser': 'src/sdk/index.ts' }, diff --git a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap index e344e1fa4f2..53786286152 100644 --- a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap +++ b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap @@ -1,6 +1,5 @@ --- source: crates/bindings/tests/deps.rs -assertion_line: 16 expression: "cargo tree -p spacetimedb -e no-dev --color never --target wasm32-unknown-unknown -f {lib}" --- total crates: 72 @@ -72,52 +71,53 @@ spacetimedb │ ├── hex │ ├── spacetimedb_bindings_macro (*) │ ├── spacetimedb_primitives (*) -│ └── spacetimedb_sats -│ ├── anyhow -│ ├── arrayvec -│ ├── bytemuck -│ ├── bytes -│ ├── chrono -│ │ └── num_traits -│ │ [build-dependencies] -│ │ └── autocfg -│ ├── decorum -│ │ ├── approx -│ │ │ └── num_traits (*) -│ │ └── num_traits (*) -│ ├── derive_more (*) -│ ├── enum_as_inner (*) -│ ├── ethnum -│ │ └── serde -│ │ └── serde_core -│ ├── hex -│ ├── itertools (*) -│ ├── lean_string -│ │ ├── castaway -│ │ │ └── rustversion -│ │ ├── itoa -│ │ └── ryu -│ ├── second_stack -│ ├── sha3 -│ │ ├── digest -│ │ │ ├── block_buffer -│ │ │ │ └── generic_array -│ │ │ │ └── typenum -│ │ │ │ [build-dependencies] -│ │ │ │ └── version_check -│ │ │ └── crypto_common -│ │ │ ├── generic_array (*) -│ │ │ └── typenum -│ │ └── keccak -│ ├── smallvec -│ ├── spacetimedb_bindings_macro (*) -│ ├── spacetimedb_primitives (*) -│ ├── thiserror -│ │ └── thiserror_impl -│ │ ├── proc_macro2 (*) -│ │ ├── quote (*) -│ │ └── syn (*) -│ └── uuid +│ ├── spacetimedb_sats +│ │ ├── anyhow +│ │ ├── arrayvec +│ │ ├── bytemuck +│ │ ├── bytes +│ │ ├── chrono +│ │ │ └── num_traits +│ │ │ [build-dependencies] +│ │ │ └── autocfg +│ │ ├── decorum +│ │ │ ├── approx +│ │ │ │ └── num_traits (*) +│ │ │ └── num_traits (*) +│ │ ├── derive_more (*) +│ │ ├── enum_as_inner (*) +│ │ ├── ethnum +│ │ │ └── serde +│ │ │ └── serde_core +│ │ ├── hex +│ │ ├── itertools (*) +│ │ ├── lean_string +│ │ │ ├── castaway +│ │ │ │ └── rustversion +│ │ │ ├── itoa +│ │ │ └── ryu +│ │ ├── second_stack +│ │ ├── sha3 +│ │ │ ├── digest +│ │ │ │ ├── block_buffer +│ │ │ │ │ └── generic_array +│ │ │ │ │ └── typenum +│ │ │ │ │ [build-dependencies] +│ │ │ │ │ └── version_check +│ │ │ │ └── crypto_common +│ │ │ │ ├── generic_array (*) +│ │ │ │ └── typenum +│ │ │ └── keccak +│ │ ├── smallvec +│ │ ├── spacetimedb_bindings_macro (*) +│ │ ├── spacetimedb_primitives (*) +│ │ ├── thiserror +│ │ │ └── thiserror_impl +│ │ │ ├── proc_macro2 (*) +│ │ │ ├── quote (*) +│ │ │ └── syn (*) +│ │ └── uuid +│ └── thiserror (*) ├── spacetimedb_primitives (*) └── spacetimedb_query_builder └── spacetimedb_lib (*) diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index 7609d9ba378..18b61f49224 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -209,13 +209,13 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs @@ -241,13 +241,13 @@ help: the trait `FilterableValue` is not implemented for `Alpha` | ^^^^^^^^^^^^ = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others = note: required for `Alpha` to implement `IndexScanRangeBounds<(Alpha,), SingleBound>` note: required by a bound in `RangedIndex::::filter` diff --git a/crates/bindings/tests/ui/views.stderr b/crates/bindings/tests/ui/views.stderr index 2b0c0dd6033..c2eb326c60c 100644 --- a/crates/bindings/tests/ui/views.stderr +++ b/crates/bindings/tests/ui/views.stderr @@ -488,13 +488,13 @@ error[E0277]: `&'a NonFilterableViewPrimaryKey` cannot appear as an argument to = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Lifecycle + &PortExposure + &PortProtocol + &RestartPolicy &TableAccess - &TableType - &bool - ðnum::int::I256 - ðnum::uint::U256 and $N others = note: required for `NonFilterableViewPrimaryKey` to implement `ViewPrimaryKeyColumn` note: required by a bound in `_::_assert_view_primary_key_column::_assert_view_primary_key_column_type` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5ea9f718833..6f951afd920 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,6 +27,7 @@ spacetimedb-codegen.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true +spacetimedb-oci = { path = "../oci" } spacetimedb-paths.workspace = true spacetimedb-schema.workspace = true @@ -59,13 +60,18 @@ serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arb serde_with = { workspace = true, features = ["chrono_0_4"] } syntect.workspace = true tabled.workspace = true +tar.workspace = true tempfile.workspace = true +sha2 = "0.10" +tokio-util.workspace = true termcolor.workspace = true termtree.workspace = true thiserror.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true toml_edit.workspace = true +url.workspace = true +uuid = { workspace = true, features = ["std", "serde", "v7"] } walkdir.workspace = true wasmbin.workspace = true webbrowser.workspace = true @@ -83,13 +89,23 @@ notify.workspace = true path-clean = "1.0.1" [dev-dependencies] +axum.workspace = true pretty_assertions.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { workspace = true } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +rustix = { version = "1", features = ["fs", "process", "termios", "event"] } + +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dev-dependencies] +rustix = { version = "1", features = ["pty"] } + [target.'cfg(windows)'.dependencies] -windows-sys = { workspace = true, features = ["Win32_System_Console"] } +windows-sys = { workspace = true, features = ["Win32_System_Console", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemServices", "Win32_System_IO", "Wdk_Storage_FileSystem"] } + +[target.'cfg(windows)'.dev-dependencies] +windows-sys = { workspace = true, features = ["Win32_System_Pipes"] } [build-dependencies] serde = { workspace = true, features = ["derive"] } diff --git a/crates/cli/docs/container-build.md b/crates/cli/docs/container-build.md new file mode 100644 index 00000000000..71749eda520 --- /dev/null +++ b/crates/cli/docs/container-build.md @@ -0,0 +1,226 @@ +# Local container image preparation + +`spacetime container build` reads one database's `container` declaration from +`spacetime.json` and prepares a verified local OCI image. It does not resolve a +database through a server or publish anything. The optional `DATABASE` argument +selects an exact local configuration target. Omit it only when the configuration +contains one container target. Container declarations do not pass to children. +The command is dispatched before saved CLI server settings or credentials are +opened; only project configuration and explicitly selected build credentials +are read. + +The same verified preparation feeds managed `publish`, described below. +Container lifecycle commands remain separate integration work. + +## Configuration + +```json +{ + "database": "example", + "container": { + "image": { + "build": { + "builder": "dockerfile", + "context": ".", + "dockerfile": "Dockerfile" + } + }, + "env_keys": ["API_KEY"], + "resources": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824, + "scratch_bytes": 1073741824, + "pids_max": 256 + } + } +} +``` + +`image` accepts exactly one source: + +- `{"build":{"context":"."}}` selects Dockerfile by default. +- `{"build":{"builder":"railpack","context":"."}}` explicitly selects Railpack. +- `{"oci_ref":"registry.example/team/image:tag"}` imports a registry image. +- `{"oci_ref":"oci:./existing-layout"}` imports a local OCI image-layout directory. + +Build contexts and local layout paths are relative to the configuration +directory. The Dockerfile path is relative to its build context. A tag is +resolved once during import; the result records the selected immutable manifest +digest. Registry import requires Skopeo. A local directory import requires no +external image tools. + +The image's `Entrypoint` followed by `Cmd` supplies the command by default. +`command` replaces the complete argv. Image `User` and `WorkingDir` are preserved +unless `user` or `working_directory` overrides them. An omitted or empty image +`WorkingDir` normalizes to `/`. Image `Env` remains in the +immutable configuration blob. `env_keys` contains runtime store references, +never runtime values. Reserved `SPACETIMEDB_` keys are rejected. Mounts must be +empty in Stage 1. Shared resource, startup-string, port, and environment-key +limits apply to the normalized specification. + +## Tools and credentials + +Source builds require an explicitly selected local BuildKit Unix socket: + +```sh +spacetime container build example \ + --project-path ./project \ + --platform linux/amd64 \ + --out-dir ./prepared-image \ + --buildkit-host unix:///absolute/path/to/disposable-buildkit.sock +``` + +Use an endpoint you own and have verified. The command never selects a saved +Docker daemon or BuildKit endpoint. This implementation invokes `buildctl` +directly and currently runs external tools on Linux and macOS. An existing local +OCI directory can also be imported without those subprocesses on Windows. +`--platform` is required and accepts `linux/amd64` or `linux/arm64`; it never +silently uses the build computer's platform. `--out-dir` must not already exist, +and its parent must exist. + +`--buildctl`, `--skopeo`, and `--railpack` select absolute executable paths or +command names on PATH. Missing tools +produce installation/path errors. No executable is automatically downloaded. +Railpack is pinned to `0.35.0`, paired with +`ghcr.io/railwayapp/railpack-frontend:v0.35.0`. Detection or version failures end +the build without changing builders. Its `prepare` output is given to BuildKit's +matching gateway frontend, following the [Railpack production integration](https://railpack.com/platforms/running-railpack-in-production). + +Registry access is anonymous unless `--registry-auth-file FILE` explicitly names +an auth JSON file. The CLI creates an isolated auth directory and does not load +saved Docker credentials. Registry credentials are distinct from Spacetime +credentials and are never copied into prepared metadata. + +Pass build secrets as `--build-secret NAME=FILE`. Files enter BuildKit's secret +interface; Railpack receives only their names during plan generation. Builds +with secrets disable cached build results. Build secrets are separate from +`env_keys`, and prebuilt imports reject them. Credential and secret files are +bounded to 1 MiB each. Subprocess environments are cleared except for basic +executable/temporary-directory paths and the explicitly isolated tool paths. +Do not put secret values in a Dockerfile, image command, or image defaults. + +## Output and failure behavior + +The new output directory contains an OCI image layout plus `prepared.json`: + +- `container` is the normalized container specification. +- `manifest` is the selected immutable manifest descriptor. +- `objects` contains the executable manifest/config/layer closure, with digest, + media type, size, purpose, and a path relative to the layout. + +Prepared metadata omits image environment values and build credentials. The +image configuration blob necessarily retains its image defaults. Consumers must +verify descriptors when reopening output; the local metadata is not an +admission receipt from a server. + +`prepare_container` returns `PreparedContainer`, which owns its temporary +layout. Dropping it removes the temporary output. `persist` transfers the verified +layout with an atomic operation that does not replace an existing output. +Credentials, plans, and intermediate builder files remain in the temporary +workspace and are removed. Failed verification never returns a prepared object. +The shared `Runner` interface permits structured fake builders in tests and +reuse by a later managed publication path. + +Verification checks manifest/config digests, platform, exact compressed layer +digests, uncompressed diff IDs, and bounded tar structure. Layers are inspected +as streams and are never unpacked into the project. Imports reject archive +links, special entries, unknown paths, and duplicate object paths. Bounds include +256 layers, 64 GiB compressed image data, 128 GiB expanded tar data, and one +million expanded entries, with the shared per-object/decompression limits. +Two image tools and two verification workers may run concurrently. Tool calls +have a 30-minute deadline; verification has a 5-minute deadline. Captured stdout +and stderr each have a 4 MiB limit and are not echoed, avoiding accidental +secret disclosure in tool diagnostics. + +Cancellation retains the tool's workspace until its Unix process group has been +signalled and its leader reaped. The group leader remains unreaped until the +signal, preventing reuse of its numeric process-group identity. This cleanup is +for locally trusted tools; it is not a sandbox for a tool that deliberately +escapes its group. BuildKit daemon work is subject to the selected daemon's own +client-disconnect cancellation and retention policies. Blocking verification +retains its worker permit and workspace while checking cancellation between +bounded reads. + +Tests use generated local OCI fixtures and fake tool invocations, including an +owned shell fixture for process cleanup. They do not execute Docker, BuildKit, +Railpack, Skopeo, or any server operation. Actual supported-builder acceptance +remains a separate integration check. + +## Managed publication + +`spacetime publish` publishes a selected target's container declaration through +managed publication. Select the image platform explicitly. Server selection uses +the normal CLI URL, configured alias, or default: + +```sh +spacetime publish my-db --server https://your-test-server.example \ + --container-platform linux/amd64 \ + --artifact-endpoint https://your-test-artifacts.example +``` + +The URLs above are placeholders. Managed transport requires HTTPS for remote +servers and also supports HTTP loopback servers. The CLI does not send the publisher's Bearer credential to an +advertised artifact origin unless it is the same origin as the selected server +or the exact URL is explicitly approved with `--artifact-endpoint`. It does not +follow HTTP redirects or inherit HTTP proxy settings for managed publication. +Image registry credentials remain separate and require `--registry-auth-file`. + +A target with a container declaration and no module source preserves its existing +module. A new container-only database uses the immutable versioned empty module. +An explicit `module-path`, `bin-path`, or `js-path` replaces the module. An omitted +container preserves it; `--remove-container` removes it. `--remove-module` selects +the empty module and runs the existing authorized migration preflight. A selected +container declaration conflicts with `--remove-container`, and a configured +module source conflicts with `--remove-module`. Manual migrations and data-clear +publication are rejected. Precompiled NativeAOT modules can use `--bin-path`; +managed source compilation with `--native-aot` is not yet supported. + +Ordinary module-only publications use the legacy path. When deployment inspection +finds an existing managed revision, module-only updates use managed publication +with `ContainerAction::Keep`. `--managed` explicitly selects managed publication +for a new module-only deployment. A managed error never falls back to a raw +module publication. Existing databases use their exact current deployment +revision as a compare-and-set precondition. + +New managed databases first reserve a server-generated Identity under the +publisher and creation options. A caller cannot select an unreserved new Identity. +`--parent` resolves an accessible existing database; `--organization` currently +requires the organization's Identity. Requested database naming runs separately +after activation. If naming fails or its response is lost, the command reports +the successfully created Identity and does not repeat publication or overwrite +names on a later resume. + +Before staging artifacts, the CLI saves a private operation directory under +`.spacetime/publications/` beside the project configuration, or the explicit +`--publication-state-dir`. The directory retains exact request bytes, immutable +module/OCI artifacts and upload receipts. It contains no publisher credential or +resolved database environment values. Image blobs can contain environment +values baked into the image, so treat the directory as private build output. + +If a request fails or the command is interrupted, use the printed directory: + +```sh +spacetime publish --resume-publication .spacetime/publications/OPERATION_UUID \ + --server https://your-test-server.example \ + --artifact-endpoint https://your-test-artifacts.example +``` + +Resume authenticates the original publisher and server, observes accepted state, +and reuses the exact operation bytes. It does not read `spacetime.json`, rebuild, +or resolve a changed image tag. Ambiguous uploads query the same upload receipt +before continuing. If current database-read access was revoked, resume uses the +exact stored PUT to recover the original publisher's admitted result before +requiring local artifacts. A lost publication response never creates a second +operation. The coordinator PUT has a bounded 30-minute timeout covering its +separate schema, image, storage, and confirmation steps; interruption retains +the journal while the outcome is uncertain. Exact public PUT replay remains +limited to the operation's seven-day retry window; server-side recovery has its +own durable lifetime. +The default activation wait is 60 seconds; `--publication-wait 0` returns after +the first confirmed status, and a pending status prints its resume instruction. +Keep the directory while an outcome is uncertain; a confirmed terminal operation +can be removed locally when its artifacts are no longer needed. + +Container declarations remain local to their exact database target. They do not +inherit to nested targets. `spacetime dev` currently rejects selected container +targets because it does not yet supervise their local runtime. diff --git a/crates/cli/docs/container-operations.md b/crates/cli/docs/container-operations.md new file mode 100644 index 00000000000..3aa680d8167 --- /dev/null +++ b/crates/cli/docs/container-operations.md @@ -0,0 +1,39 @@ +# Container status and lifecycle requests + +`spacetime container status DATABASE` reads the current control state without +opening database storage. Viewer access is sufficient. `--json` returns the +typed status, including desired and observed state, deployment revision, +generation, fixed diagnostics, and public endpoint availability. An absent +current instance is distinct from an old instance's terminal report. Endpoint +allocation may be pending while the rest of the status is available. + +`spacetime container start DATABASE`, `stop DATABASE`, and `restart DATABASE` +require Admin access. Start requests execution, stop requests a stop, and restart +requests a new instance with a fresh environment snapshot. Acceptance records +the desired action; physical stop and readiness complete asynchronously. Use +status to inspect progress. None of these commands changes the published image +or its declaration. + +Each lifecycle request uses a UUIDv7. Before sending it, the CLI prints structured +retry parameters to stderr: the action, resolved database Identity, request ID, +and selected server URL. Keep these parameters if the response is lost. Retry +with that Identity, the same action, `--request-id UUID`, and `--server URL`. +`--request-id` requires an Identity so a changed database name cannot redirect a +retry. Never create a fresh request ID merely to resolve an unknown outcome. + +An accepted exact retry returns the original result generation, even if another +request has since advanced the database generation. Reusing an ID for another +action fails. Current Admin access is checked again on every retry. The retry +window is seven days from the UUID's timestamp. `--json` writes only a verified +receipt to stdout; its generation is a decimal string to preserve all 64 bits. + +Server selection and login use the normal CLI configuration, with an explicit +`--server` override supported on every command. Authenticated requests disable +redirects and inherited proxies. Public `container url` remains anonymous, and +local `container build` continues to run without loading server credentials. + +The HTTP counterparts are authenticated `GET +/v1/database/DATABASE/container/status` and `POST` to the `start`, `stop`, or +`restart` suffix with `{"request_id":"UUID"}`. Successful mutations return HTTP +202. Responses are not cacheable. The operational API rejects hosted container +credentials and uses current ordinary database roles. diff --git a/crates/cli/src/container/config.rs b/crates/cli/src/container/config.rs new file mode 100644 index 00000000000..5a5de9f8a3c --- /dev/null +++ b/crates/cli/src/container/config.rs @@ -0,0 +1,162 @@ +//! Per-database declarations. Values and build credentials are never part of this configuration. +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerMount, ContainerPort, ContainerResources, ContainerSpec, ContainerSpecLimits, + ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema, MAX_ENV_VARS}; +use spacetimedb_oci::ContainerConfig as ImageConfig; +use std::collections::BTreeMap; +use std::path::PathBuf; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfig { + pub image: ImageSource, + pub command: Option>, + pub user: Option, + pub working_directory: Option, + #[serde(default = "default_mode")] + pub mode: ContainerMode, + #[serde(default = "default_restart")] + pub restart: RestartPolicy, + #[serde(default)] + pub env_keys: Vec, + /// Declaration schema for a database without a user module. Values are + /// supplied by the ordinary publish env configuration and shell override. + #[serde(default)] + pub env_schema: Option>, + pub resources: ContainerResources, + #[serde(default)] + pub ports: Vec, + #[serde(default)] + pub mounts: Vec, + #[serde(default = "default_grace")] + pub stop_grace_ms: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentDeclarationConfig { + #[serde(default)] + pub optional: bool, + /// Omission accepts any string. One value is a literal constraint; several + /// values are a finite union. An empty list is invalid. + pub values: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ImageSource { + Build(BuildImage), + Prebuilt(PrebuiltImage), +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BuildImage { + #[serde(deserialize_with = "source_build")] + pub build: SourceBuild, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrebuiltImage { + pub oci_ref: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "builder", rename_all = "lowercase", deny_unknown_fields)] +pub enum SourceBuild { + Dockerfile { + #[serde(default = "default_context")] + context: PathBuf, + #[serde(default = "default_dockerfile")] + dockerfile: PathBuf, + }, + Railpack { + #[serde(default = "default_context")] + context: PathBuf, + }, +} + +// A missing builder is Dockerfile. Keep deserialization strict after applying +// that one default, including rejection of Dockerfile-only fields on Railpack. +fn source_build<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + let mut value = serde_json::Value::deserialize(deserializer)?; + if let Some(object) = value.as_object_mut() { + object.entry("builder").or_insert_with(|| "dockerfile".into()); + } + serde_json::from_value(value).map_err(serde::de::Error::custom) +} +fn default_context() -> PathBuf { + PathBuf::from(".") +} +fn default_dockerfile() -> PathBuf { + PathBuf::from("Dockerfile") +} +fn default_mode() -> ContainerMode { + ContainerMode::Service +} +fn default_restart() -> RestartPolicy { + RestartPolicy::OnFailure +} +fn default_grace() -> u32 { + 30_000 +} + +impl ContainerConfig { + pub fn environment_schema(&self) -> Result> { + let Some(schema) = &self.env_schema else { + return Ok(None); + }; + ensure!( + schema.len() <= MAX_ENV_VARS, + "too many container environment declarations" + ); + let declarations = schema + .iter() + .map(|(name, declaration)| EnvironmentDeclaration { + name: name.clone(), + optional: declaration.optional, + constraint: match declaration.values.as_deref() { + None => EnvironmentConstraint::AnyString, + Some([literal]) => EnvironmentConstraint::Literal(literal.clone()), + Some(values) => EnvironmentConstraint::OneOf(values.to_vec()), + }, + }) + .collect(); + Ok(Some(EnvironmentSchema::new(declarations)?)) + } + + pub fn normalize( + &self, + manifest: OciDigest, + platform: ImagePlatform, + image: &ImageConfig, + ) -> Result { + ensure!(self.mounts.is_empty(), "container mounts are not supported in Stage 1"); + self.environment_schema()?; + let argv = image.argv(self.command.as_deref())?; + spacetimedb_lib::container::validate_exec_size(&argv, image.env.as_deref().unwrap_or_default())?; + Ok(ContainerSpec { + image_manifest: manifest, + image_platform: platform, + argv, + user: self.user.clone().unwrap_or_else(|| image.user.clone()), + working_directory: self.working_directory.clone().unwrap_or_else(|| { + if image.working_directory.is_empty() { + "/".into() + } else { + image.working_directory.clone() + } + }), + mode: self.mode, + restart: self.restart, + env_keys: self.env_keys.clone(), + resources: self.resources, + ports: self.ports.clone(), + mounts: vec![], + stop_grace_ms: self.stop_grace_ms, + } + .normalize(&ContainerSpecLimits::default())?) + } +} diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs new file mode 100644 index 00000000000..bfb9accfc2d --- /dev/null +++ b/crates/cli/src/container/mod.rs @@ -0,0 +1,400 @@ +//! Local image preparation shared by build-only commands and managed publication. +pub mod config; +pub mod oci; +pub mod process; +pub mod publish; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{ensure, Context, Result}; +use config::{ContainerConfig, ImageSource, SourceBuild}; +use oci::{LocalArtifact, VerifiedImage}; +use process::{Invocation, Runner}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ContainerSpec, ImagePlatform}; +use spacetimedb_oci::Descriptor; +use std::{ + ffi::OsString, + fs, + io::Read, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub const RAILPACK_VERSION: &str = "0.35.0"; +pub const RAILPACK_FRONTEND: &str = "ghcr.io/railwayapp/railpack-frontend:v0.35.0"; +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const VERIFY_TIMEOUT: Duration = Duration::from_secs(5 * 60); +static VERIFIERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(2))); + +pub struct BuildSecret { + pub name: String, + pub file: PathBuf, +} +pub struct BuildTools { + pub buildctl: PathBuf, + pub buildkit_host: Option, + pub railpack: PathBuf, + pub skopeo: PathBuf, + pub registry_auth_file: Option, + pub secrets: Vec, +} +impl Default for BuildTools { + fn default() -> Self { + Self { + buildctl: "buildctl".into(), + buildkit_host: None, + railpack: "railpack".into(), + skopeo: "skopeo".into(), + registry_auth_file: None, + secrets: vec![], + } + } +} + +/// Public metadata contains immutable descriptors and relative paths, not image +/// environment values or credentials. Reopening output must verify the bytes. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedMetadata { + pub version: u32, + pub container: ContainerSpec, + pub manifest: Descriptor, + pub objects: Vec, +} +/// Owns all temporary output. Drop removes it; persist transfers only verified +/// artifacts. A failed/cancelled prepare never yields this type. +pub struct PreparedContainer { + workspace: Arc, + pub metadata: PreparedMetadata, +} +impl PreparedContainer { + pub fn layout(&self) -> PathBuf { + self.workspace.path().join("verified") + } + pub fn persist(self, output: &Path) -> Result<()> { + ensure!( + !output.exists(), + "output directory already exists: {}", + output.display() + ); + // The final transition must not replace a directory created after the + // initial check. The workspace is placed alongside the requested output. + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::renameat_with( + rustix::fs::CWD, + self.layout(), + rustix::fs::CWD, + output, + rustix::fs::RenameFlags::NOREPLACE, + ) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(windows)] + fs::rename(self.layout(), output) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + anyhow::bail!("atomic container output is supported on Linux, macOS, and Windows"); + Ok(()) + } +} +struct CancelOnDrop(CancellationToken); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} +fn path_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("{key}={path}").into()) +} +fn csv_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("\"{key}={}\"", path.replace('"', "\"\""))) +} +fn read_credential(path: &Path) -> Result> { + let mut bytes = vec![]; + fs::File::open(path)?.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= 1024 * 1024, + "explicit build credential file exceeds 1 MiB" + ); + Ok(bytes) +} +fn local_buildkit_host(value: Option<&str>) -> Result<&str> { + let value = value.context("source builds require --buildkit-host unix:///absolute/path/to/buildkitd.sock")?; + let path = value + .strip_prefix("unix://") + .context("local source builds require an explicit BuildKit Unix socket")?; + ensure!( + Path::new(path).is_absolute() && !path.contains(['\0', '\n', '\r']), + "invalid local BuildKit socket path" + ); + Ok(value) +} + +pub async fn prepare_container( + declaration: &ContainerConfig, + config_dir: &Path, + platform: ImagePlatform, + tools: &BuildTools, + workspace_parent: &Path, + runner: &impl Runner, + cancel: CancellationToken, +) -> Result { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "choose linux/amd64 or linux/arm64 explicitly" + ); + ensure!( + declaration.mounts.is_empty(), + "container mounts are not supported in Stage 1" + ); + let cancel = cancel.child_token(); + let _cancel = CancelOnDrop(cancel.clone()); + let mut workspace = tempfile::Builder::new(); + workspace.prefix(".spacetime-image-"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + // Registry credentials and build secrets are copied here before a + // publication journal exists, including under an existing public base. + workspace.permissions(fs::Permissions::from_mode(0o700)); + } + let workspace = Arc::new(workspace.tempdir_in(workspace_parent)?); + let base = workspace.path(); + fs::create_dir(base.join("auth"))?; + let auth_file = base.join("auth/config.json"); + let auth = tools + .registry_auth_file + .as_deref() + .map(read_credential) + .transpose()? + .unwrap_or_else(|| br#"{"auths":{}}"#.to_vec()); + fs::write(&auth_file, auth)?; + let environment = vec![ + ("DOCKER_CONFIG".into(), base.join("auth").into_os_string()), + ("REGISTRY_AUTH_FILE".into(), auth_file.clone().into_os_string()), + ("XDG_CONFIG_HOME".into(), base.join("tool-config").into_os_string()), + ("XDG_CACHE_HOME".into(), base.join("tool-cache").into_os_string()), + ]; + let invoke = |tool: PathBuf, label, args, cwd: PathBuf| Invocation { + tool, + label, + args, + env: environment.clone(), + cwd, + workspace: workspace.clone(), + timeout: BUILD_TIMEOUT, + cancel: cancel.clone(), + }; + let input = base.join("input"); + let mut archive = None; + match &declaration.image { + ImageSource::Prebuilt(image) => { + ensure!( + tools.secrets.is_empty(), + "build secrets cannot be supplied with a prebuilt image" + ); + if let Some(path) = image.oci_ref.strip_prefix("oci:") { + let path = config_dir + .join(path) + .canonicalize() + .context("prebuilt OCI layout does not exist")?; + ensure!(path.is_dir(), "oci: must name an OCI layout directory"); + return verify(declaration.clone(), platform, workspace.clone(), path, None, cancel).await; + } + ensure!( + !image.oci_ref.is_empty() + && !image.oci_ref.starts_with('-') + && !image.oci_ref.contains(['\0', '\n', '\r']), + "invalid OCI registry reference" + ); + let reference = image.oci_ref.strip_prefix("docker://").unwrap_or(&image.oci_ref); + ensure!( + !reference.contains("://") + && (!reference.contains('@') + || reference.rsplit_once('@').is_some_and(|(_, digest)| digest + .parse::() + .is_ok())), + "invalid OCI registry reference" + ); + let args = vec![ + "--override-os".into(), + platform.os.clone().into(), + "--override-arch".into(), + platform.architecture.clone().into(), + "copy".into(), + "--preserve-digests".into(), + "--authfile".into(), + auth_file.into_os_string(), + format!("docker://{reference}").into(), + format!("oci:{}:prepared", input.display()).into(), + ]; + runner + .run(invoke( + tools.skopeo.clone(), + "Skopeo image import", + args, + config_dir.to_path_buf(), + )) + .await?; + } + ImageSource::Build(image) => { + let endpoint = local_buildkit_host(tools.buildkit_host.as_deref())?; + let (context, dockerfile, railpack) = match &image.build { + SourceBuild::Dockerfile { context, dockerfile } => (context, Some(dockerfile), false), + SourceBuild::Railpack { context } => (context, None, true), + }; + let context = config_dir + .join(context) + .canonicalize() + .context("build context does not exist")?; + ensure!(context.is_dir(), "build context must be a directory"); + let mut secret_arguments = vec![]; + let mut secret_names = std::collections::BTreeSet::new(); + for (index, secret) in tools.secrets.iter().enumerate() { + spacetimedb_lib::container::validate_env_key(&secret.name)?; + ensure!(secret_names.insert(&secret.name), "duplicate build secret name"); + let path = base.join(format!("secret-{index}")); + fs::write(&path, read_credential(&secret.file)?)?; + secret_arguments.extend([ + OsString::from("--secret"), + format!("id={},{}", secret.name, csv_argument("src", &path)?).into(), + ]); + } + let dockerfile = if railpack { + let version = runner + .run(invoke( + tools.railpack.clone(), + "Railpack version check", + vec!["--version".into()], + context.clone(), + )) + .await?; + let version = std::str::from_utf8(&version.stdout).context("invalid Railpack version response")?; + ensure!( + version + .split_whitespace() + .any(|word| word.trim_start_matches('v') == RAILPACK_VERSION), + "install Railpack {RAILPACK_VERSION} to match the pinned frontend" + ); + let plan = base.join("railpack-plan.json"); + let mut args = vec![ + "prepare".into(), + context.clone().into_os_string(), + "--plan-out".into(), + plan.clone().into_os_string(), + "--info-out".into(), + base.join("railpack-info.json").into_os_string(), + ]; + // Only names enter the plan; BuildKit receives the actual files. + for name in secret_names { + args.extend(["--env".into(), format!("{name}=").into()]); + } + runner + .run(invoke( + tools.railpack.clone(), + "Railpack detection", + args, + context.clone(), + )) + .await?; + ensure!( + plan.is_file() && plan.metadata()?.len() <= 4 * 1024 * 1024, + "Railpack did not produce a bounded build plan" + ); + plan + } else { + context + .join(dockerfile.unwrap()) + .canonicalize() + .context("Dockerfile does not exist")? + }; + ensure!( + dockerfile.is_file(), + "Dockerfile or Railpack plan must be a regular file" + ); + let output = base.join("image.tar"); + let mut args = vec![ + "--addr".into(), + endpoint.into(), + "build".into(), + "--frontend".into(), + if railpack { + "gateway.v0".into() + } else { + "dockerfile.v0".into() + }, + "--local".into(), + path_argument("context", &context)?, + "--local".into(), + path_argument("dockerfile", dockerfile.parent().unwrap())?, + "--opt".into(), + path_argument("filename", Path::new(dockerfile.file_name().unwrap()))?, + "--opt".into(), + format!("platform={}/{}", platform.os, platform.architecture).into(), + "--output".into(), + format!("type=oci,{}", csv_argument("dest", &output)?).into(), + ]; + if railpack { + args.extend(["--opt".into(), format!("source={RAILPACK_FRONTEND}").into()]); + } + if !secret_arguments.is_empty() { + args.push("--no-cache".into()); + args.extend(secret_arguments); + } + runner + .run(invoke(tools.buildctl.clone(), "BuildKit OCI build", args, context)) + .await?; + archive = Some(output); + } + } + verify(declaration.clone(), platform, workspace.clone(), input, archive, cancel).await +} + +async fn verify( + declaration: ContainerConfig, + platform: ImagePlatform, + workspace: Arc, + input: PathBuf, + archive: Option, + cancel: CancellationToken, +) -> Result { + let permit = VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI images are already being verified")?; + let owner = workspace.clone(); + let metadata = tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + VERIFY_TIMEOUT; + if let Some(archive) = archive { + oci::extract_archive(&archive, &input, &cancel, deadline)?; + } + let VerifiedImage { + manifest, + config, + objects, + } = oci::verify_layout(&input, &owner.path().join("verified"), &platform, &cancel, deadline)?; + let container = declaration.normalize(manifest.digest, platform, &config.config)?; + let metadata = PreparedMetadata { + version: 1, + container, + manifest, + objects, + }; + fs::write( + owner.path().join("verified/prepared.json"), + serde_json::to_vec_pretty(&metadata)?, + )?; + Ok::<_, anyhow::Error>(metadata) + }) + .await + .context("OCI verification worker stopped")??; + Ok(PreparedContainer { workspace, metadata }) +} diff --git a/crates/cli/src/container/oci.rs b/crates/cli/src/container/oci.rs new file mode 100644 index 00000000000..3e5ec888a57 --- /dev/null +++ b/crates/cli/src/container/oci.rs @@ -0,0 +1,262 @@ +//! Bounded OCI import. Layers are inspected as streams, never unpacked into the project. +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest}; +use spacetimedb_oci::{ + self as oci, + layers::{verify_layer_with_check, LayerLimits}, + Descriptor, ImageConfig, +}; +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::{Read, Seek, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +pub const MAX_EXPANDED_IMAGE_BYTES: u64 = 128 * 1024 * 1024 * 1024; +const MAX_ARCHIVE_ENTRIES: usize = 1024; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactKind { + Manifest, + Config, + Layer, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LocalArtifact { + pub kind: ArtifactKind, + pub descriptor: Descriptor, + /// Relative to the owned OCI layout directory. + pub path: PathBuf, +} +pub(crate) struct VerifiedImage { + pub manifest: Descriptor, + pub config: ImageConfig, + pub objects: Vec, +} + +pub(crate) fn check(cancel: &CancellationToken, deadline: Instant) -> std::io::Result<()> { + if cancel.is_cancelled() || Instant::now() >= deadline { + Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "container preparation cancelled or timed out", + )) + } else { + Ok(()) + } +} +fn blob_path(digest: OciDigest) -> PathBuf { + PathBuf::from("blobs/sha256").join(digest.to_string().strip_prefix("sha256:").expect("SHA-256 digest")) +} +fn bounded_file(path: &Path, limit: u64) -> Result { + ensure!( + fs::symlink_metadata(path)?.is_file(), + "OCI object must be a regular file" + ); + let file = File::open(path)?; + ensure!(file.metadata()?.len() <= limit, "OCI object exceeds its size bound"); + Ok(file) +} +fn read_small(path: &Path, limit: usize) -> Result> { + let mut bytes = vec![]; + bounded_file(path, limit as u64)? + .take(limit as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= limit, "OCI metadata exceeds its size bound"); + Ok(bytes) +} + +pub(crate) fn extract_archive( + archive: &Path, + output: &Path, + cancel: &CancellationToken, + deadline: Instant, +) -> Result<()> { + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut seen = BTreeSet::new(); + let mut total = 0u64; + let archive = bounded_file(archive, oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024)?; + for (index, entry) in tar::Archive::new(archive).entries()?.enumerate() { + check(cancel, deadline)?; + ensure!(index < MAX_ARCHIVE_ENTRIES, "too many OCI archive entries"); + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let name = path.to_str().context("OCI archive path is not UTF-8")?; + if entry.header().entry_type().is_dir() { + ensure!( + matches!(name.trim_end_matches('/'), "." | "blobs" | "blobs/sha256"), + "unexpected OCI archive directory" + ); + continue; + } + ensure!( + entry.header().entry_type().is_file(), + "OCI archive links and special files are unsupported" + ); + let blob = name.strip_prefix("blobs/sha256/"); + ensure!( + matches!(name, "index.json" | "oci-layout") + || blob.is_some_and( + |v| v.len() == 64 && v.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()) + ), + "unexpected OCI archive path" + ); + ensure!(seen.insert(path.clone()), "duplicate OCI archive path"); + let size = entry.size(); + total = total.checked_add(size).context("OCI archive size overflow")?; + ensure!( + total <= oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024, + "OCI archive exceeds image size bound" + ); + if blob.is_none() { + ensure!(size <= oci::MAX_MANIFEST_BYTES as u64, "OCI archive metadata too large"); + } + let mut file = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = entry.read(&mut buffer)?; + if n == 0 { + break; + } + file.write_all(&buffer[..n])?; + } + } + Ok(()) +} + +pub(crate) fn verify_layout( + input: &Path, + output: &Path, + platform: &ImagePlatform, + cancel: &CancellationToken, + deadline: Instant, +) -> Result { + check(cancel, deadline)?; + let layout: serde_json::Value = serde_json::from_slice(&read_small(&input.join("oci-layout"), 1024)?)?; + ensure!( + layout.get("imageLayoutVersion").and_then(|v| v.as_str()) == Some("1.0.0"), + "unsupported OCI layout version" + ); + let index = read_small(&input.join("index.json"), oci::MAX_MANIFEST_BYTES)?; + // Layout metadata is not a hashed image object and may omit mediaType. + // Defaults apply only here, never to immutable registry index bytes. + let mut index: serde_json::Value = serde_json::from_slice(&index)?; + index + .as_object_mut() + .context("OCI layout index must be an object")? + .entry("mediaType") + .or_insert_with(|| oci::OCI_INDEX.into()); + let index = serde_json::to_vec(&index)?; + let parsed: oci::ImageIndex = serde_json::from_slice(&index)?; + ensure!( + parsed.media_type == oci::OCI_INDEX, + "unsupported OCI layout index media type" + ); + ensure!( + parsed.schema_version == 2 && parsed.manifests.len() <= oci::MAX_INDEX_ENTRIES, + "invalid OCI layout index" + ); + // A local layout index commonly names one manifest without platform metadata. + // The immutable image config below must still match the explicit platform. + let descriptor = if parsed.manifests.len() == 1 && parsed.manifests[0].platform.is_none() { + parsed.manifests.into_iter().next().unwrap() + } else { + oci::select_platform(&index, platform)? + }; + // A layout can name a multi-platform index; select exactly one executable manifest. + let source = read_small(&input.join(blob_path(descriptor.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&descriptor, &source)?; + let (descriptor, bytes) = if matches!(descriptor.media_type.as_str(), oci::OCI_INDEX | oci::DOCKER_INDEX) { + let selected = oci::select_platform(&source, platform)?; + let bytes = read_small(&input.join(blob_path(selected.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&selected, &bytes)?; + (selected, bytes) + } else { + (descriptor, source) + }; + ensure!( + matches!(descriptor.media_type.as_str(), oci::OCI_MANIFEST | oci::DOCKER_MANIFEST), + "OCI layout does not select an executable image" + ); + let manifest = oci::parse_manifest(&bytes)?; + let config_bytes = read_small(&input.join(blob_path(manifest.config.digest)), oci::MAX_CONFIG_BYTES)?; + let config = oci::parse_config(&config_bytes, &manifest, platform)?; + let mut compressed = 0u64; + let mut expanded = 0u64; + let mut entries = 0u64; + for (layer, diff_id) in manifest.layers.iter().zip(&config.rootfs.diff_ids) { + check(cancel, deadline)?; + compressed = compressed.checked_add(layer.size).context("image size overflow")?; + ensure!(compressed <= oci::MAX_IMAGE_BYTES, "compressed image exceeds bound"); + let file = bounded_file(&input.join(blob_path(layer.digest)), layer.size)?; + let size = verify_layer_with_check(file, layer, *diff_id, LayerLimits::default(), || { + check(cancel, deadline) + })?; + expanded = expanded + .checked_add(size.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(size.entries) + .context("image entry count overflow")?; + ensure!( + expanded <= MAX_EXPANDED_IMAGE_BYTES && entries <= 1_000_000, + "expanded image exceeds bound" + ); + } + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut objects = vec![]; + for object in oci::object_closure(descriptor.clone(), &manifest)? { + check(cancel, deadline)?; + let path = blob_path(object.digest); + let mut source = bounded_file(&input.join(&path), object.size)?; + source.rewind()?; + let mut target = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = source.read(&mut buffer)?; + if n == 0 { + break; + } + total = total.checked_add(n as u64).context("object size overflow")?; + ensure!(total <= object.size, "OCI object changed while copying"); + hash.update(&buffer[..n]); + target.write_all(&buffer[..n])?; + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "OCI object changed while copying" + ); + objects.push(LocalArtifact { + kind: if object.digest == descriptor.digest { + ArtifactKind::Manifest + } else if object.digest == manifest.config.digest { + ArtifactKind::Config + } else { + ArtifactKind::Layer + }, + descriptor: object, + path, + }); + } + fs::write(output.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#)?; + fs::write( + output.join("index.json"), + serde_json::to_vec( + &serde_json::json!({"schemaVersion":2,"mediaType":oci::OCI_INDEX,"manifests":[descriptor]}), + )?, + )?; + Ok(VerifiedImage { + manifest: descriptor, + config, + objects, + }) +} diff --git a/crates/cli/src/container/process.rs b/crates/cli/src/container/process.rs new file mode 100644 index 00000000000..3d548833178 --- /dev/null +++ b/crates/cli/src/container/process.rs @@ -0,0 +1,212 @@ +//! Local trusted tools run in a dedicated Unix process group. Cancellation +//! retains the workspace until that group is signalled and its leader reaped. +//! This is process cleanup, not containment of tools that deliberately escape +//! their process group. +use anyhow::Result; +use std::{ffi::OsString, future::Future, path::PathBuf, sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub struct Invocation { + pub tool: PathBuf, + pub label: &'static str, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: PathBuf, + pub workspace: Arc, + pub timeout: Duration, + pub cancel: CancellationToken, +} +pub struct Output { + pub stdout: Vec, +} +pub trait Runner: Sync { + fn run(&self, invocation: Invocation) -> impl Future> + Send; +} +pub struct LocalRunner; + +impl Runner for LocalRunner { + async fn run(&self, invocation: Invocation) -> Result { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + local::run(invocation).await + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = invocation; + anyhow::bail!( + "local image tools currently require Linux or macOS; import an existing oci: directory instead" + ) + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local { + use super::*; + use anyhow::{bail, ensure, Context}; + use rustix::process::{Pid, WaitIdOptions}; + use std::process::Stdio; + use tokio::{ + io::AsyncReadExt, + process::Command, + sync::{oneshot, Semaphore}, + }; + + static PROCESSES: std::sync::LazyLock> = std::sync::LazyLock::new(|| Arc::new(Semaphore::new(2))); + const MAX_OUTPUT: usize = 4 * 1024 * 1024; + + async fn read_bounded(mut reader: impl tokio::io::AsyncRead + Unpin) -> Result> { + let mut bytes = vec![]; + (&mut reader) + .take(MAX_OUTPUT as u64 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= MAX_OUTPUT, "builder diagnostic output exceeded 4 MiB"); + Ok(bytes) + } + + // Keep the group leader unreaped until after killpg. Its waitable PID pins + // the numeric process-group identity even when it exits before descendants. + struct ProcessGroup(Option); + impl ProcessGroup { + fn kill(&mut self) -> std::io::Result<()> { + if let Some(pid) = self.0.take() { + match rustix::process::kill_process_group(pid, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => (), + #[cfg(target_os = "macos")] + Err(rustix::io::Errno::PERM) if exited_leader_is_sole_member(pid) => (), + Err(error) => return Err(error.into()), + } + } + Ok(()) + } + async fn observe_exit(&mut self) -> Result<()> { + let pid = self.0.context("local image tool ownership lost")?; + loop { + match rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ) { + Ok(Some(_)) => return Ok(()), + Ok(None) | Err(rustix::io::Errno::INTR) => (), + Err(error) => { + // An external reaper invalidates numeric PID ownership. + // Never signal that group after this boundary. + if error == rustix::io::Errno::CHILD { + self.0 = None; + } + return Err(error).context("could not observe local image tool exit"); + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } + + #[cfg(target_os = "macos")] + fn exited_leader_is_sole_member(pid: Pid) -> bool { + // XNU excludes zombies from killpg's eligible processes, and reports + // EPERM when only an unreaped leader remains. Do not generalize EPERM: + // prove this exact child has exited and its group contains only it. + // WNOWAIT keeps the PID/PGID pinned across the bounded inventory. + if !matches!( + rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ), + Ok(Some(_)) + ) { + return false; + } + #[link(name = "proc")] + unsafe extern "C" { + fn proc_listpgrppids( + pgrpid: std::ffi::c_int, + buffer: *mut std::ffi::c_void, + buffersize: std::ffi::c_int, + ) -> std::ffi::c_int; + } + // libproc returns the number of PIDs copied, including zombies. Two + // slots distinguish the sole leader from any extra/truncated members. + let mut members = [0i32; 2]; + // SAFETY: writable aligned storage and its exact byte size are passed; + // the group ID belongs to the still-unreaped child observed above. + let count = unsafe { + proc_listpgrppids( + pid.as_raw_pid(), + members.as_mut_ptr().cast(), + std::mem::size_of_val(&members) as std::ffi::c_int, + ) + }; + count == 1 && members[0] == pid.as_raw_pid() + } + impl Drop for ProcessGroup { + fn drop(&mut self) { + let _ = self.kill(); + } + } + + pub(super) async fn run(invocation: Invocation) -> Result { + let permit = PROCESSES + .clone() + .try_acquire_owned() + .context("two local image tools are already running")?; + let (mut send, receive) = oneshot::channel(); + tokio::spawn(async move { + let _permit = permit; + let result = async { + ensure!(!send.is_closed() && !invocation.cancel.is_cancelled(), "container build cancelled"); + let _workspace = invocation.workspace; + let mut command = Command::new(&invocation.tool); + command.args(&invocation.args).current_dir(&invocation.cwd).env_clear(); + // No implicit registry, Spacetime, proxy or builder credentials. + for key in ["PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP"] { + if let Some(value) = std::env::var_os(key) { command.env(key, value); } + } + command.envs(invocation.env).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()).process_group(0); + let mut child = command.spawn().with_context(|| format!("could not run {}; install it or provide its executable path", invocation.tool.display()))?; + let mut group = ProcessGroup(Some(Pid::from_raw(child.id().context("builder PID unavailable")? as i32).context("invalid builder PID")?)); + let stdout = child.stdout.take().context("builder stdout unavailable")?; + let stderr = child.stderr.take().context("builder stderr unavailable")?; + let outcome = { + let completion = async { + let (status, stdout, _) = tokio::try_join!( + async { + group.observe_exit().await?; + group.kill().context("failed to stop builder descendants")?; + Ok::<_, anyhow::Error>(child.wait().await?) + }, + read_bounded(stdout), + read_bounded(stderr), + )?; + ensure!(status.success(), "{} failed ({status}); no prepared output was accepted", invocation.label); + Ok(Output { stdout }) + }; + tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("container build caller closed")), + _ = invocation.cancel.cancelled() => Err(anyhow::anyhow!("container build cancelled")), + _ = tokio::time::sleep(invocation.timeout) => Err(anyhow::anyhow!("{} exceeded its build deadline", invocation.label)), + result = completion => result, + } + }; + // Disarmed before every reap, including completion above. Child + // wait is cached if completion already reaped it. + group.kill().context("failed to stop local image tool group")?; + match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { + Ok(result) => { result.context("local image tool could not be reaped")?; }, + Err(_) => { + // The owner retains the workspace and permit throughout + // delayed physical cleanup, even if the caller is gone. + child.wait().await.context("local image tool could not be reaped")?; + bail!("local image tool required delayed physical cleanup"); + } + } + outcome + }.await; + let _ = send.send(result); + }); + receive.await.context("local image tool owner stopped")? + } +} diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs new file mode 100644 index 00000000000..4bc85b5206a --- /dev/null +++ b/crates/cli/src/container/publish/client.rs @@ -0,0 +1,503 @@ +//! Ordinary publisher transport. Credentials are held in memory and never +//! redirected, inherited from a proxy, or sent to an unapproved artifact URL. +use anyhow::{bail, ensure, Context, Result}; +use reqwest::{header::HeaderValue, Client, Method, StatusCode, Url}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use spacetimedb_client_api_messages::name::{DomainName, PrePublishResult, SetDomainsResult}; +use spacetimedb_lib::{container::OciDigest, deployment::api::*, Identity, Uuid}; +use std::time::Duration; + +pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024; +// A submission composes the 120s schema worker, 300s image verification, +// two independently bounded 300s artifact writes, and read/pin/control calls. +// This caller deadline does not extend those server resource or credential +// bounds. Cancellation/timeout leaves the exact operation in its journal. +const COORDINATOR_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); +pub const UPLOAD_CHUNK_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +#[error("{action} returned HTTP {status}")] +pub struct HttpFailure { + pub action: &'static str, + pub status: StatusCode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UploadKind { + Manifest, + Config, + Layer, + Module, +} +impl UploadKind { + fn header(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Config => "config", + Self::Layer => "layer", + Self::Module => "module", + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObjectRef { + pub digest: OciDigest, + pub size: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadStatus { + pub id: uuid::Uuid, + pub object: ObjectRef, + pub offset: u64, + pub expires_at: u64, + pub complete: bool, +} +impl UploadStatus { + pub fn validate(&self, object: ObjectRef, id: Option) -> Result<()> { + ensure!( + self.id.get_version_num() == 4 && id.is_none_or(|id| self.id == id), + "artifact upload session changed" + ); + ensure!( + self.object == object && self.offset <= object.size && (!self.complete || self.offset == object.size), + "artifact upload descriptor or offset changed" + ); + Ok(()) + } +} + +pub struct ArtifactEndpoint(Url); +impl ArtifactEndpoint { + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +pub struct PublisherClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl PublisherClient { + pub fn new(server: &str, mut authorization: HeaderValue) -> Result { + let server = endpoint(server)?; + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "managed publication requires ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + pub fn server(&self) -> &Url { + &self.server + } + fn request(&self, method: Method, url: Url) -> reqwest::RequestBuilder { + self.http + .request(method, url) + .header(reqwest::header::AUTHORIZATION, self.authorization.clone()) + } + /// A response cannot grant itself permission to receive this credential. + /// Same-origin paths are accepted; another origin requires an explicit URL. + pub fn artifact_endpoint(&self, advertised: &str, approved: Option<&str>) -> Result { + let artifact = endpoint(advertised)?; + if artifact.origin() != self.server.origin() { + let approved=approved.context("artifact service uses another origin; pass --artifact-endpoint with its exact trusted URL before sending publisher credentials")?; + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } else if let Some(approved) = approved { + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } + Ok(ArtifactEndpoint(artifact)) + } + pub async fn capabilities(&self) -> Result> { + let response = self + .http + .get(route(&self.server, &["v1", "containers", "capabilities"])?) + .send() + .await?; + optional_json(response, "publication capabilities").await + } + pub async fn permission(&self) -> Result { + json( + self.request( + Method::GET, + route(&self.server, &["v1", "containers", "publish-permission"])?, + ) + .send() + .await?, + "publication permission", + ) + .await + } + pub async fn deployment(&self, name: &str) -> Result> { + optional_json( + self.request( + Method::GET, + route(&self.server, &["v1", "database", name, "deployment"])?, + ) + .send() + .await?, + "deployment inspection", + ) + .await + } + /// Keep inspects the existing metadata API, not a code-download permission. + /// Identity and program hash bind its declarations to the previously + /// observed deployment; the later revision/operation CAS closes the race. + pub async fn selected_environment( + &self, + prior: &DeploymentStatus, + ) -> Result { + const MAX_SCHEMA_BYTES: usize = 16 * 1024 * 1024; + let mut response = self + .request( + Method::GET, + route( + &self.server, + &["v1", "database", &prior.database_identity.to_hex(), "schema"], + )?, + ) + .query(&[("version", "10")]) + .send() + .await?; + ensure!( + response.status() == StatusCode::OK, + "selected module schema was not authorized or available" + ); + let hash = match &prior.deployment.current().module { + spacetimedb_lib::deployment::ModuleComponent::User(module) => module.program_hash, + spacetimedb_lib::deployment::ModuleComponent::SystemEmpty(module) => module.program_hash, + }; + ensure!( + response + .headers() + .get("x-spacetimedb-database-identity") + .and_then(|h| h.to_str().ok()) + == Some(prior.database_identity.to_hex().as_str()), + "selected module database identity changed" + ); + ensure!( + response + .headers() + .get("x-spacetimedb-module-hash") + .and_then(|h| h.to_str().ok()) + == Some(hash.to_string().as_str()), + "selected module program hash changed" + ); + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_SCHEMA_BYTES as u64), + "selected module schema exceeds its bound" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_SCHEMA_BYTES.saturating_sub(bytes.len()), + "selected module schema exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + let spacetimedb_lib::sats::serde::SerdeWrapper(raw) = serde_json::from_slice::< + spacetimedb_lib::sats::serde::SerdeWrapper, + >(&bytes) + .map_err(|_| anyhow::anyhow!("invalid selected module schema"))?; + let module = spacetimedb_schema::def::ModuleDef::try_from(spacetimedb_lib::RawModuleDef::V10(raw)) + .map_err(|_| anyhow::anyhow!("invalid selected module schema"))?; + Ok(module.environment().clone()) + } + pub async fn reserve(&self, request: &ReserveDatabaseRequest) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "containers", "reservations"])?, + ) + .json(request) + .send() + .await?, + "database reservation", + ) + .await + } + pub async fn submit(&self, database: Identity, request: &PublishRequest) -> Result { + let bytes = serde_json::to_vec(request)?; + self.submit_bytes(database, &bytes).await + } + /// Resume sends these same immutable bytes, without reconstructing a request + /// from a changed tag, project configuration, or current deployment. + pub async fn submit_bytes(&self, database: Identity, bytes: &[u8]) -> Result { + ensure!( + bytes.len() <= MAX_PUBLISH_REQUEST_BYTES, + "publication HTTP body exceeds its bound" + ); + PublishRequest::decode(bytes)?; + json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "deployment"])?, + ) + .timeout(COORDINATOR_REQUEST_TIMEOUT) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(bytes.to_vec()) + .send() + .await?, + "publication submission", + ) + .await + } + pub async fn status(&self, database: Identity, operation: Uuid) -> Result> { + optional_json( + self.request( + Method::GET, + route( + &self.server, + &[ + "v1", + "database", + &database.to_hex(), + "deployment", + "operations", + &operation.to_string(), + ], + )?, + ) + .send() + .await?, + "publication status", + ) + .await + } + pub async fn preflight(&self, database: Identity, kind: &str, bytes: &[u8]) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "database", &database.to_hex(), "pre_publish"])?, + ) + .query(&[("host_type", kind), ("pretty_print_style", "NoColor")]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await?, + "module migration preflight", + ) + .await + } + pub async fn set_name(&self, database: Identity, name: &str) -> Result<()> { + let name: DomainName = name.parse()?; + let result: SetDomainsResult = json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "names"])?, + ) + .json(&[name]) + .send() + .await?, + "database naming", + ) + .await?; + ensure!( + matches!(result, SetDomainsResult::Success), + "database was created, but assigning its requested name was not confirmed" + ); + Ok(()) + } + pub async fn begin_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + kind: UploadKind, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::POST, + route(&artifact.0, &["v1", "databases", &database.to_hex(), "uploads"])?, + ) + .header("x-spacetimedb-artifact-kind", kind.header()) + .json(&serde_json::json!({"kind":kind,"object":object})) + .send() + .await?, + "artifact upload creation", + ) + .await?; + status.validate(object, None)?; + Ok(status) + } + pub async fn upload_status( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + id: uuid::Uuid, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::GET, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &id.to_string()], + )?, + ) + .send() + .await?, + "artifact upload status", + ) + .await?; + status.validate(object, Some(id))?; + Ok(status) + } + pub async fn append_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + chunk: Vec, + ) -> Result { + ensure!( + !chunk.is_empty() + && chunk.len() <= UPLOAD_CHUNK_BYTES + && status + .offset + .checked_add(chunk.len() as u64) + .is_some_and(|end| end <= status.object.size), + "invalid artifact chunk" + ); + let next: UploadStatus = json( + self.request( + Method::PATCH, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &status.id.to_string()], + )?, + ) + .query(&[("offset", status.offset)]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(chunk) + .send() + .await?, + "artifact upload append", + ) + .await?; + next.validate(status.object, Some(status.id))?; + Ok(next) + } + pub async fn complete_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + ) -> Result { + let completed: ObjectRef = json( + self.request( + Method::POST, + route( + &artifact.0, + &[ + "v1", + "databases", + &database.to_hex(), + "uploads", + &status.id.to_string(), + "complete", + ], + )?, + ) + .send() + .await?, + "artifact upload completion", + ) + .await?; + // The completion endpoint confirms the immutable object, rather than + // returning session status. The route already binds the original UUID; + // retain that session only after checking the exact digest and size. + ensure!(completed == status.object, "artifact completion descriptor changed"); + let mut next = status.clone(); + next.offset = completed.size; + next.complete = true; + next.validate(status.object, Some(status.id))?; + Ok(next) + } +} + +pub(crate) fn is_loopback(url: &Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + Some(url::Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + None => false, + } +} +pub fn endpoint(value: &str) -> Result { + let mut url = Url::parse(value).context("resolved server URL must use HTTP(S)")?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "endpoint must not contain credentials, query, or fragment" + ); + let local = is_loopback(&url); + ensure!( + url.scheme() == "https" || (url.scheme() == "http" && local), + "use HTTPS, or HTTP for a local loopback server" + ); + ensure!(url.host_str().is_some(), "endpoint host is missing"); + if !url.path().ends_with('/') { + let path = format!("{}/", url.path()); + url.set_path(&path); + } + Ok(url) +} +fn route(base: &Url, segments: &[&str]) -> Result { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid base URL"))? + .pop_if_empty() + .extend(segments); + Ok(url) +} +async fn optional_json(response: reqwest::Response, action: &'static str) -> Result> { + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + json(response, action).await.map(Some) +} +async fn json(mut response: reqwest::Response, action: &'static str) -> Result { + if !response.status().is_success() { + return Err(HttpFailure { + action, + status: response.status(), + } + .into()); + } + if response + .content_length() + .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64) + { + bail!("{action} response exceeds its bound"); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(bytes.len()), + "{action} response exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid {action} response")) +} diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs new file mode 100644 index 00000000000..a93e8caf8ce --- /dev/null +++ b/crates/cli/src/container/publish/journal.rs @@ -0,0 +1,392 @@ +//! Progress metadata is separate from the protected immutable submission body. +//! Only that owner-readable body contains the complete resolved environment. +mod protected; +use super::client::{ObjectRef, UploadKind, UploadStatus}; +use crate::container::{oci, PreparedContainer}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::{ + container::OciDigest, + deployment::{api::*, PUBLISH_PROTOCOL_VERSION}, + Identity, Uuid, +}; +use std::{ + fs::{self, File}, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +const MAX_RECORD_BYTES: usize = 1024 * 1024; +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadRecord { + pub kind: UploadKind, + pub object: ObjectRef, + pub session: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Record { + pub version: u32, + pub server: String, + pub artifact_endpoint: String, + pub publisher: Identity, + pub database: Option, + pub reservation: Option, + pub requested_name: Option, + /// UTF-8 JSON sent verbatim on every submission attempt. + #[serde(skip)] + pub request_json: String, + /// Private local integrity check only; never included in a wire request. + pub request_digest: OciDigest, + pub uploads: Vec, + pub submitted: bool, + pub status: Option, + pub name_confirmed: bool, + pub naming_attempted: bool, +} +impl Record { + pub fn request(&self) -> Result { + ensure!( + self.version == 2 && self.request_json.len() <= MAX_PUBLISH_REQUEST_BYTES, + "unsupported or oversized publication resume record" + ); + ensure!( + spacetimedb_oci::sha256(self.request_json.as_bytes()) == self.request_digest, + "publication request bytes changed" + ); + let request = PublishRequest::decode(self.request_json.as_bytes())?; + let envelope = &request.manifest.current().envelope; + ensure!( + envelope.version == PUBLISH_PROTOCOL_VERSION, + "unsupported publication protocol" + ); + ensure!( + envelope.operation_id.get_version() == Some(spacetimedb_lib::sats::uuid::Version::V7), + "publication operation must be UUIDv7" + ); + ensure!(self.uploads.len() <= 259, "too many retained publication objects"); + if let Some(reservation) = &self.reservation { + ensure!( + reservation.operation_id == envelope.operation_id + && reservation.version == PUBLISH_PROTOCOL_VERSION + && request.creation.as_ref() == Some(&reservation.options), + "reservation differs from the immutable publication request" + ); + } else { + ensure!( + self.database.is_some() && request.creation.is_none(), + "publication database binding is missing" + ); + } + for upload in &self.uploads { + ensure!(upload.object.size > 0, "empty publication artifact"); + if let Some(status) = &upload.session { + status.validate(upload.object, None)?; + } + } + if let Some(status) = &self.status { + self.check_status(&request, status)?; + } + Ok(request) + } + pub fn check_status(&self, request: &PublishRequest, status: &PublicationStatus) -> Result<()> { + let envelope = &request.manifest.current().envelope; + ensure!( + Some(status.database_identity) == self.database + && status.operation_id == envelope.operation_id + && status.expected_revision == envelope.expected_revision + && status.expected_last_operation == envelope.expected_last_operation + && status.publication_epoch != 0 + && self + .status + .as_ref() + .is_none_or(|previous| previous.publication_epoch == status.publication_epoch) + && status.proposed_revision == request.manifest.current().deployment.revision()?, + "publication response belongs to another operation or deployment" + ); + Ok(()) + } +} + +pub struct Journal { + directory: PathBuf, + _parents: Vec, + _lock: File, + pub record: Record, +} +struct IncompleteDirectory(Option); +impl Drop for IncompleteDirectory { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = fs::remove_dir_all(path); + } + } +} +impl Journal { + /// Prepare ancestors before local image preparation. Keep the returned + /// directory handles alive while preparing or accessing retained inputs. + pub(crate) fn prepare_base(base: &Path) -> Result> { + let mut directories = fs::DirBuilder::new(); + directories.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + // New ancestors must protect retained environment values even + // when the user's umask permits group or other-user writes. + directories.mode(0o700); + } + directories.create(base)?; + protected::pin_parents(base) + } + + pub fn create( + base: &Path, + record: Record, + image: Option, + module: Option<&[u8]>, + ) -> Result { + let request = record.request()?; + let has_image = image.is_some(); + let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); + let _base_parents = Self::prepare_base(base)?; + protected::create_directory(&directory) + .context("cannot create protected publication directory; resume an existing operation instead")?; + let mut incomplete = IncompleteDirectory(Some(directory.clone())); + let parents = protected::pin_parents(&directory)?; + let lock = Self::lock(&directory, true)?; + let mut submission = protected::file(&directory.join("submission.json"), true, true)?; + submission.write_all(record.request_json.as_bytes())?; + submission.sync_all()?; + if let Some(image) = image { + image.persist(&directory.join("image"))?; + } + if let Some(module) = module { + let mut file = File::options() + .write(true) + .create_new(true) + .open(directory.join("module.blob"))?; + file.write_all(module)?; + file.sync_all()?; + } + let journal = Self { + directory, + _parents: parents, + _lock: lock, + record, + }; + journal.sync_retained_artifacts(has_image)?; + journal.save()?; + sync_directory_chain(&journal.directory)?; + incomplete.0 = None; + Ok(journal) + } + pub fn open(directory: &Path) -> Result { + let parents = protected::pin_parents(directory)?; + protected::directory(directory)?; + let directory = directory + .canonicalize() + .context("publication resume directory not found")?; + let lock = Self::lock(&directory, false)?; + let path = directory.join("publication.json"); + let mut bytes = vec![]; + protected::file(&path, false, false)? + .take(MAX_RECORD_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let mut record: Record = + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid publication progress record"))?; + record.request_json = String::from_utf8(read_submission(&directory)?) + .map_err(|_| anyhow::anyhow!("invalid protected publication body"))?; + let request = record.request()?; + ensure!( + directory.file_name().and_then(|name| name.to_str()) + == Some(request.manifest.current().envelope.operation_id.to_string().as_str()), + "publication directory belongs to another operation" + ); + // Creation saves JSON only after artifact flush. Reconfirm the parent + // link if its creator stopped before finishing that last barrier. + sync_directory_chain(&directory)?; + Ok(Self { + directory, + _parents: parents, + _lock: lock, + record, + }) + } + fn sync_retained_artifacts(&self, has_image: bool) -> Result<()> { + let mut files = self + .record + .uploads + .iter() + .map(|upload| self.object_path(upload)) + .collect::>(); + if has_image { + files.extend( + ["oci-layout", "index.json", "prepared.json"].map(|name| self.directory.join("image").join(name)), + ); + } + let mut directories = std::collections::BTreeSet::new(); + for path in files { + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + // Write access also lets Windows flush an owned immutable artifact. + File::options().read(true).write(true).open(&path)?.sync_all()?; + let mut parent = path.parent(); + while let Some(path) = parent { + directories.insert(path.to_owned()); + if path == self.directory { + break; + } + parent = path.parent(); + } + } + // Flush children before the directories that link them. + for directory in directories.into_iter().rev() { + sync_directory(&directory)?; + } + Ok(()) + } + fn lock(directory: &Path, create: bool) -> Result { + protected::directory(directory)?; + let file = protected::file(&directory.join("publication.lock"), create, true)?; + file.try_lock() + .context("another process is using this publication resume directory")?; + Ok(file) + } + pub fn directory(&self) -> &Path { + &self.directory + } + pub fn operation(&self) -> Result { + Ok(self.record.request()?.manifest.current().envelope.operation_id) + } + /// Check the on-disk body even on status-only recovery: missing or altered + /// retained input must never silently become an empty environment. + pub fn submission_bytes(&self) -> Result> { + self.record.request()?; + let bytes = read_submission(&self.directory)?; + ensure!( + bytes == self.record.request_json.as_bytes(), + "protected publication body changed" + ); + Ok(bytes) + } + pub fn save(&self) -> Result<()> { + self.submission_bytes()?; + let bytes = serde_json::to_vec_pretty(&self.record)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + #[cfg(not(windows))] + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + #[cfg(windows)] + let mut file = protected::temporary(&self.directory)?; + protected::verify_file(file.as_file())?; + file.write_all(&bytes)?; + file.as_file().sync_all()?; + file.persist(self.directory.join("publication.json"))?; + #[cfg(unix)] + File::open(&self.directory)?.sync_all()?; + Ok(()) + } + pub fn object_path(&self, upload: &UploadRecord) -> PathBuf { + match upload.kind { + UploadKind::Module => self.directory.join("module.blob"), + _ => self.directory.join("image/blobs/sha256").join( + upload + .object + .digest + .to_string() + .strip_prefix("sha256:") + .expect("SHA256"), + ), + } + } + /// Reopening state never trusts stale local object bytes. This repeats only + /// their digest check; server preparation remains the admission authority. + pub async fn verify_objects(&self, cancel: CancellationToken) -> Result<()> { + let permit = crate::container::VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI verifications are already running")?; + let objects = self + .record + .uploads + .iter() + .map(|upload| (self.object_path(upload), upload.object)) + .collect::>(); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + crate::container::VERIFY_TIMEOUT; + for (path, object) in objects { + oci::check(&cancel, deadline)?; + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + let mut file = File::open(path)?; + ensure!(file.metadata()?.len() == object.size, "retained artifact size changed"); + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + oci::check(&cancel, deadline)?; + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + total = total.checked_add(read as u64).context("artifact size overflow")?; + ensure!(total <= object.size, "retained artifact grew"); + hash.update(&buffer[..read]); + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "retained publication artifact digest changed" + ); + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("publication verification worker stopped")??; + Ok(()) + } +} + +fn sync_directory(directory: &Path) -> Result<()> { + #[cfg(unix)] + File::open(directory)?.sync_all()?; + // As in paths::utils::write_atomic, Windows directory handles cannot be + // synced through std. File flushes still precede publishing the journal. + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} +fn sync_directory_chain(directory: &Path) -> Result<()> { + let canonical = directory.canonicalize()?; + for ancestor in canonical.ancestors() { + sync_directory(ancestor)?; + } + Ok(()) +} + +fn read_submission(directory: &Path) -> Result> { + protected::directory(directory)?; + let mut bytes = Vec::new(); + protected::file(&directory.join("submission.json"), false, false)? + .take(MAX_PUBLISH_REQUEST_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_PUBLISH_REQUEST_BYTES, + "protected publication body exceeds its bound" + ); + Ok(bytes) +} diff --git a/crates/cli/src/container/publish/journal/protected.rs b/crates/cli/src/container/publish/journal/protected.rs new file mode 100644 index 00000000000..4df45fe46d1 --- /dev/null +++ b/crates/cli/src/container/publish/journal/protected.rs @@ -0,0 +1,91 @@ +//! Files containing resolved values are opened without following links and +//! checked through the opened handle. Directory ownership is checked before +//! any access; progress rewrites inherit the same private directory policy. +#[cfg(unix)] +use anyhow::{ensure, Result}; +#[cfg(unix)] +use std::{ + fs::{self, File}, + path::Path, +}; + +#[cfg(unix)] +pub(super) fn create_directory(path: &Path) -> Result<()> { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new().mode(0o700).create(path)?; + directory(path) +} + +#[cfg(unix)] +pub(super) fn directory(path: &Path) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + ensure!(metadata.is_dir(), "publication directory must be a private directory"); + private_metadata(&metadata) +} + +#[cfg(unix)] +fn private_metadata(metadata: &fs::Metadata) -> Result<()> { + use std::os::unix::fs::MetadataExt; + ensure!( + metadata.uid() == rustix::process::geteuid().as_raw() && metadata.mode() & 0o077 == 0, + "publication storage must be owned by the current user and inaccessible to other users" + ); + Ok(()) +} + +#[cfg(unix)] +pub(super) fn verify_file(file: &File) -> Result<()> { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + ensure!( + metadata.is_file() && metadata.nlink() == 1, + "publication storage must be a regular file without additional links" + ); + private_metadata(&metadata) +} + +#[cfg(unix)] +pub(super) fn file(path: &Path, create: bool, write: bool) -> Result { + use std::os::unix::fs::OpenOptionsExt; + let file = File::options() + .read(true) + .write(write) + .create_new(create) + .mode(0o600) + .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) + .open(path)?; + verify_file(&file)?; + Ok(file) +} + +/// Reject ancestors through which another user could replace a directory +/// component between checks and access. Root-owned sticky temporary roots are +/// safe: entries below them are themselves required to be root/current-owned. +#[cfg(unix)] +pub(super) fn pin_parents(path: &Path) -> Result> { + use std::os::unix::fs::MetadataExt; + let absolute = std::path::absolute(path)?; + for component in absolute.ancestors() { + let metadata = fs::symlink_metadata(component)?; + ensure!( + metadata.uid() == 0 || metadata.uid() == rustix::process::geteuid().as_raw(), + "publication path has an untrusted owner" + ); + if metadata.is_dir() { + ensure!( + metadata.mode() & 0o022 == 0 || (metadata.uid() == 0 && metadata.mode() & 0o1000 != 0), + "publication path has an untrusted writable ancestor" + ); + } else { + ensure!(metadata.is_symlink(), "invalid publication directory ancestor"); + // Check the resolved target as well as the named ancestor chain. + pin_parents(&component.canonicalize()?)?; + } + } + Ok(Vec::new()) +} + +#[cfg(windows)] +mod windows; +#[cfg(windows)] +pub(super) use windows::{create_directory, directory, file, pin_parents, temporary, verify_file}; diff --git a/crates/cli/src/container/publish/journal/protected/windows.rs b/crates/cli/src/container/publish/journal/protected/windows.rs new file mode 100644 index 00000000000..755f0db3ee7 --- /dev/null +++ b/crates/cli/src/container/publish/journal/protected/windows.rs @@ -0,0 +1,334 @@ +//! Owner-only DACLs are installed atomically at directory creation, without an +//! inheritable parent ACL. Files inherit only that owner ACE. Reopening checks +//! the actual handle's owner, DACL, file type and link count before reading. +use anyhow::{ensure, Result}; +use std::{ + ffi::c_void, + fs::File, + os::windows::{ + ffi::OsStrExt, + fs::OpenOptionsExt, + io::{AsRawHandle, FromRawHandle, OwnedHandle}, + }, + path::Path, + ptr::null_mut, +}; +use windows_sys::Win32::{ + Foundation::{LocalFree, GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, + Security::{ + Authorization::{ + ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, GetSecurityInfo, + SDDL_REVISION_1, SE_FILE_OBJECT, + }, + EqualSid, GetAce, GetSecurityDescriptorControl, GetTokenInformation, TokenUser, ACCESS_ALLOWED_ACE, ACE_HEADER, + DACL_SECURITY_INFORMATION, INHERIT_ONLY_ACE, OWNER_SECURITY_INFORMATION, SECURITY_ATTRIBUTES, + SE_DACL_PROTECTED, TOKEN_QUERY, TOKEN_USER, + }, + Storage::FileSystem::{ + CreateDirectoryW, CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, CREATE_NEW, + FILE_ALL_ACCESS, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, READ_CONTROL, + }, + System::{ + SystemServices::ACCESS_ALLOWED_ACE_TYPE, + Threading::{GetCurrentProcess, OpenProcessToken}, + }, +}; + +struct LocalAllocation(*mut c_void); +impl Drop for LocalAllocation { + fn drop(&mut self) { + // SAFETY: this pointer is allocated by a successful Windows security + // descriptor/SID conversion API and is freed exactly once. + unsafe { + LocalFree(self.0); + } + } +} + +fn check(ok: i32) -> Result<()> { + ensure!(ok != 0, "protected publication storage Windows operation failed"); + Ok(()) +} + +/// Aligned backing storage owns the TOKEN_USER and its SID together. +fn user() -> Result> { + unsafe { + let mut raw = null_mut(); + check(OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut raw))?; + let token = OwnedHandle::from_raw_handle(raw); + let mut length = 0; + GetTokenInformation(token.as_raw_handle(), TokenUser, null_mut(), 0, &mut length); + ensure!( + (size_of::()..=1024).contains(&(length as usize)), + "invalid Windows user identity size" + ); + let mut data = vec![0usize; (length as usize).div_ceil(size_of::())]; + check(GetTokenInformation( + token.as_raw_handle(), + TokenUser, + data.as_mut_ptr().cast(), + length, + &mut length, + ))?; + Ok(data) + } +} + +fn owner_descriptor() -> Result { + unsafe { + let user = user()?; + let sid = (*(user.as_ptr().cast::())).User.Sid; + let mut sid_string = null_mut(); + check(ConvertSidToStringSidW(sid, &mut sid_string))?; + let _sid_string = LocalAllocation(sid_string.cast()); + // Windows SID strings are bounded by the SID representation. Read only + // through the terminating NUL in the returned allocation. + let mut length = 0; + while *sid_string.add(length) != 0 { + length += 1; + } + let sid = String::from_utf16(std::slice::from_raw_parts(sid_string, length))?; + let sddl: Vec<_> = format!("O:{sid}D:P(A;OICI;FA;;;{sid})") + .encode_utf16() + .chain([0]) + .collect(); + let mut descriptor = null_mut(); + check(ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + SDDL_REVISION_1, + &mut descriptor, + null_mut(), + ))?; + Ok(LocalAllocation(descriptor)) + } +} + +pub(in super::super) fn create_directory(path: &Path) -> Result<()> { + unsafe { + let descriptor = owner_descriptor()?; + let attributes = SECURITY_ATTRIBUTES { + nLength: size_of::() as u32, + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + let path_wide: Vec<_> = path.as_os_str().encode_wide().chain([0]).collect(); + ensure!( + !path_wide[..path_wide.len() - 1].contains(&0), + "invalid publication path" + ); + check(CreateDirectoryW(path_wide.as_ptr(), &attributes))?; + } + directory(path) +} + +pub(in super::super) fn directory(path: &Path) -> Result<()> { + let file = File::options() + .access_mode(READ_CONTROL) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + verify(&file, true) +} + +pub(in super::super) fn file(path: &Path, create: bool, write: bool) -> Result { + let file = if create { + // Explicit owner is necessary even with a private parent: elevated + // Windows tokens can default newly created files to Administrators. + unsafe { + let descriptor = owner_descriptor()?; + let attributes = SECURITY_ATTRIBUTES { + nLength: size_of::() as u32, + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + let path_wide: Vec<_> = path.as_os_str().encode_wide().chain([0]).collect(); + ensure!( + !path_wide[..path_wide.len() - 1].contains(&0), + "invalid publication path" + ); + let raw = CreateFileW( + path_wide.as_ptr(), + GENERIC_READ | if write { GENERIC_WRITE } else { 0 }, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &attributes, + CREATE_NEW, + FILE_FLAG_OPEN_REPARSE_POINT, + null_mut(), + ); + ensure!(raw != INVALID_HANDLE_VALUE, "cannot create protected publication file"); + File::from_raw_handle(raw) + } + } else { + File::options() + .read(true) + .write(write) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)? + }; + verify_file(&file)?; + Ok(file) +} + +pub(in super::super) fn temporary(directory: &Path) -> Result { + let path = directory.join(format!(".publication-{}.tmp", uuid::Uuid::now_v7())); + let file = file(&path, true, true)?; + Ok(tempfile::NamedTempFile::from_parts( + file, + tempfile::TempPath::from_path(path), + )) +} + +pub(in super::super) fn verify_file(file: &File) -> Result<()> { + verify(file, false) +} + +fn verify(file: &File, directory: bool) -> Result<()> { + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = std::mem::zeroed(); + check(GetFileInformationByHandle(file.as_raw_handle(), &mut info))?; + ensure!( + info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 + && (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0) == directory + && (directory || info.nNumberOfLinks == 1), + "publication storage must not contain reparse points or additional file links" + ); + let mut owner = null_mut(); + let mut dacl = null_mut(); + let mut descriptor = null_mut(); + ensure!( + GetSecurityInfo( + file.as_raw_handle(), + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + &mut owner, + null_mut(), + &mut dacl, + null_mut(), + &mut descriptor + ) == 0, + "cannot verify protected publication storage permissions" + ); + let _descriptor = LocalAllocation(descriptor); + let user = user()?; + let sid = (*(user.as_ptr().cast::())).User.Sid; + ensure!( + !owner.is_null() && EqualSid(owner, sid) != 0 && !dacl.is_null() && (*dacl).AceCount == 1, + "publication storage must allow only its current owner" + ); + let mut ace = null_mut(); + check(GetAce(dacl, 0, &mut ace))?; + let header = &*ace.cast::(); + ensure!( + header.AceType as u32 == ACCESS_ALLOWED_ACE_TYPE + && header.AceSize as usize >= size_of::(), + "invalid publication access entry" + ); + let ace = &*ace.cast::(); + ensure!( + ace.Header.AceType as u32 == ACCESS_ALLOWED_ACE_TYPE + && ace.Header.AceFlags as u32 & INHERIT_ONLY_ACE == 0 + && ace.Mask == FILE_ALL_ACCESS + && EqualSid((&raw const ace.SidStart).cast_mut().cast(), sid) != 0, + "publication storage must allow only its current owner" + ); + if directory { + let mut control = 0; + let mut revision = 0; + check(GetSecurityDescriptorControl(descriptor, &mut control, &mut revision))?; + ensure!( + control & SE_DACL_PROTECTED != 0, + "publication directory must exclude inherited permissions" + ); + } + } + Ok(()) +} + +/// Deny DELETE sharing for every directory component for the entire journal +/// lifetime. Even a writable parent then cannot rename/replace an opened child +/// while secret files are accessed by path. Reparse ancestors are rejected. +pub(in super::super) fn pin_parents(path: &Path) -> Result> { + use windows_sys::Win32::Storage::FileSystem::FILE_READ_ATTRIBUTES; + let absolute = std::path::absolute(path)?; + let mut files = Vec::new(); + for path in absolute.ancestors().collect::>().into_iter().rev() { + let file = File::options() + .access_mode(FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = std::mem::zeroed(); + check(GetFileInformationByHandle(file.as_raw_handle(), &mut info))?; + ensure!( + info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 + && info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0, + "publication directory ancestors must not be reparse points" + ); + } + files.push(file); + } + Ok(files) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owner_only_directory_and_files_survive_reopen_and_reject_extra_links() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + drop(file(&body, false, false).unwrap()); + let progress = temporary(&path).unwrap(); + verify_file(progress.as_file()).unwrap(); + std::fs::hard_link(&body, path.join("extra-link")).unwrap(); + assert!(file(&body, false, false).is_err()); + } + + #[test] + fn reparse_points_are_rejected_before_reading() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + let link = path.join("link"); + std::os::windows::fs::symlink_file(&body, &link).unwrap(); + assert!(file(&link, false, false).is_err()); + let link = root.path().join("linked-directory"); + std::os::windows::fs::symlink_dir(&path, &link).unwrap(); + assert!(directory(&link).is_err()); + } + + #[test] + fn opening_a_file_with_a_permissive_dacl_fails_closed() { + use windows_sys::Win32::Security::{Authorization::SetNamedSecurityInfoW, PROTECTED_DACL_SECURITY_INFORMATION}; + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + let path_wide: Vec<_> = body.as_os_str().encode_wide().chain([0]).collect(); + // A NULL DACL grants everyone access. This is an owned empty fixture; + // no secret bytes are written before or after changing its ACL. + unsafe { + assert_eq!( + SetNamedSecurityInfoW( + path_wide.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + null_mut(), + null_mut(), + null_mut(), + null_mut() + ), + 0 + ); + } + assert!(file(&body, false, false).is_err()); + } +} diff --git a/crates/cli/src/container/publish/mod.rs b/crates/cli/src/container/publish/mod.rs new file mode 100644 index 00000000000..0cf686cec9b --- /dev/null +++ b/crates/cli/src/container/publish/mod.rs @@ -0,0 +1,316 @@ +//! Managed publication uses one immutable operation and a durable local resume +//! directory. Ambiguous HTTP responses never select a new operation or fall +//! back to the legacy module publication endpoint. +pub mod client; +pub mod journal; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{bail, ensure, Context, Result}; +use client::{ArtifactEndpoint, PublisherClient, UploadStatus, UPLOAD_CHUNK_BYTES}; +use journal::Journal; +use spacetimedb_lib::deployment::{ + api::{PublicationPhase, PublicationStatus}, + operation_expiry_ms, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug)] +pub enum Outcome { + Complete(PublicationStatus), + Pending(PublicationStatus), + Aborted(PublicationStatus), + /// Database creation succeeded. Never describe a naming failure as a + /// rollback or send another publication operation to compensate for it. + NamingUnconfirmed(PublicationStatus), +} + +pub async fn run( + client: &PublisherClient, + journal: &mut Journal, + approved_artifact: Option<&str>, + wait: Duration, + cancel: CancellationToken, +) -> Result { + let cancel = cancel.child_token(); + let _cancel = crate::container::CancelOnDrop(cancel.clone()); + ensure!( + client.server().as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let artifact = client.artifact_endpoint(&journal.record.artifact_endpoint, approved_artifact)?; + journal.submission_bytes()?; + let request = journal.record.request()?; + let operation = request.manifest.current().envelope.operation_id; + let permission = client.permission().await?; + ensure!( + permission.identity == journal.record.publisher, + "resume publisher differs from the original publication identity" + ); + + // Observe an already admitted operation before requiring original local + // object files or applying today's new-publication permission/limits. + if journal.record.submitted + && let Some(database) = journal.record.database + { + match observe_or_replay(client, journal, database, operation).await { + Ok(Some(status)) => { + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + return wait_and_name(client, journal, wait, &cancel).await; + } + Ok(None) => (), + Err(error) => { + return Err(error).context("publication observation was not confirmed; resume the same directory") + } + } + } + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; + operation_expiry_ms(operation, now)?; + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + if journal.record.database.is_none() { + let reservation = client + .reserve( + journal + .record + .reservation + .as_ref() + .context("creation reservation is missing")?, + ) + .await?; + ensure!( + reservation.operation_id == operation && reservation.staging_open, + "reservation was not confirmed for this operation" + ); + let returned = client.artifact_endpoint(&reservation.artifact_endpoint, approved_artifact)?; + ensure!( + returned.as_str() == artifact.as_str(), + "reservation changed the selected artifact endpoint" + ); + journal.record.database = Some(reservation.database_identity); + journal.save()?; + } + journal.verify_objects(cancel.clone()).await?; + for index in 0..journal.record.uploads.len() { + upload(client, &artifact, journal, index, &cancel).await?; + } + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + // This marker must be durable before sending the admission request. A crash + // after the server commits but before the response can then observe/replay. + journal.record.submitted = true; + journal.save()?; + let status = client + .submit_bytes(journal.record.database.unwrap(), &journal.submission_bytes()?) + .await + .context("publication outcome is not confirmed; resume the same directory")?; + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + wait_and_name(client, journal, wait, &cancel).await +} + +/// Operation inspection has current database-read authorization. Exact PUT +/// replay instead authenticates the original admitted publisher and immutable +/// request. It must precede local artifact/expiry work after read revocation. +async fn observe_or_replay( + client: &PublisherClient, + journal: &Journal, + database: spacetimedb_lib::Identity, + operation: spacetimedb_lib::Uuid, +) -> Result> { + match client.status(database, operation).await { + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.status == reqwest::StatusCode::FORBIDDEN) => + { + client + .submit_bytes(database, &journal.submission_bytes()?) + .await + .map(Some) + .context("exact publication replay was not confirmed; keep the same resume directory") + } + result => result, + } +} + +async fn upload( + client: &PublisherClient, + artifact: &ArtifactEndpoint, + journal: &mut Journal, + index: usize, + cancel: &CancellationToken, +) -> Result<()> { + let database = journal.record.database.context("reservation missing")?; + let item = journal.record.uploads[index].clone(); + let mut status = if let Some(prior) = item.session { + match client.upload_status(artifact, database, prior.id, item.object).await { + Ok(status) => status, + Err(error) + if error.downcast_ref::().is_some_and(|error| { + matches!(error.status, reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::GONE) + }) => + { + client.begin_upload(artifact, database, item.kind, item.object).await? + } + Err(error) => return Err(error), + } + } else { + client.begin_upload(artifact, database, item.kind, item.object).await? + }; + store_upload(journal, index, &status)?; + if status.complete { + return Ok(()); + } + let mut file = tokio::fs::File::open(journal.object_path(&journal.record.uploads[index])).await?; + let mut ambiguous_attempts = 0; + while status.offset < status.object.size { + ensure!( + !cancel.is_cancelled(), + "artifact upload cancelled; resume directory retained" + ); + file.seek(std::io::SeekFrom::Start(status.offset)).await?; + let len = usize::try_from((status.object.size - status.offset).min(UPLOAD_CHUNK_BYTES as u64))?; + let mut bytes = vec![0; len]; + file.read_exact(&mut bytes) + .await + .context("retained artifact was truncated")?; + let expected = status.offset + len as u64; + match client.append_upload(artifact, database, &status, bytes).await { + Ok(next) => { + ensure!( + next.offset == expected, + "artifact append acknowledged an unexpected offset" + ); + status = next; + ambiguous_attempts = 0; + } + Err(error) => { + if !retryable(&error) { + return Err(error); + } + ambiguous_attempts += 1; + ensure!( + ambiguous_attempts <= 3, + "artifact append outcome remains unknown; resume the same directory" + ); + let next = client + .upload_status(artifact, database, status.id, status.object) + .await?; + ensure!( + next.offset == status.offset || next.offset == expected, + "artifact offset changed outside this append" + ); + status = next; + } + } + store_upload(journal, index, &status)?; + } + status = match client.complete_upload(artifact, database, &status).await { + Ok(status) => status, + Err(error) if retryable(&error) => { + let observed = client + .upload_status(artifact, database, status.id, status.object) + .await?; + if observed.complete { + observed + } else { + return Err(error).context("artifact completion is not confirmed; resume the same directory"); + } + } + Err(error) => return Err(error), + }; + store_upload(journal, index, &status) +} +fn store_upload(journal: &mut Journal, index: usize, status: &UploadStatus) -> Result<()> { + journal.record.uploads[index].session = Some(status.clone()); + journal.save() +} +fn retryable(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some_and(|error| error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()) + || error.downcast_ref::().is_some_and(|error| { + error.status.is_server_error() + || matches!( + error.status, + reqwest::StatusCode::CONFLICT | reqwest::StatusCode::TOO_MANY_REQUESTS + ) + }) +} +async fn wait_and_name( + client: &PublisherClient, + journal: &mut Journal, + wait: Duration, + cancel: &CancellationToken, +) -> Result { + let deadline = tokio::time::Instant::now() + wait; + let request = journal.record.request()?; + loop { + let status = journal + .record + .status + .clone() + .context("confirmed publication status missing")?; + match status.phase { + PublicationPhase::AbortedBeforeCommit => return Ok(Outcome::Aborted(status)), + PublicationPhase::Complete => { + if let Some(name) = journal + .record + .requested_name + .clone() + .filter(|_| !journal.record.name_confirmed) + { + // Existing name replacement has no CAS token. Do not retry an + // ambiguous name update and overwrite aliases added later. + if journal.record.naming_attempted { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.naming_attempted = true; + journal.save()?; + if client.set_name(status.database_identity, &name).await.is_err() { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.name_confirmed = true; + journal.save()?; + } + return Ok(Outcome::Complete(status)); + } + _ => (), + } + if tokio::time::Instant::now() >= deadline { + return Ok(Outcome::Pending(status)); + } + tokio::select! { + _=cancel.cancelled()=>bail!("publication wait cancelled; its durable operation continues, resume the same directory"), + _=tokio::time::sleep_until((tokio::time::Instant::now()+Duration::from_secs(1)).min(deadline))=>(), + } + let next = tokio::time::timeout_at( + deadline, + observe_or_replay(client, journal, status.database_identity, status.operation_id), + ) + .await; + match next { + Ok(Ok(Some(next))) => { + journal.record.check_status(&request, &next)?; + journal.record.status = Some(next); + journal.save()?; + } + Ok(Ok(None)) => bail!("confirmed publication disappeared; keep the same operation and resume directory"), + Ok(Err(error)) => { + return Err(error).context("publication status is not confirmed; resume the same directory") + } + Err(_) => return Ok(Outcome::Pending(status)), + } + } +} diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs new file mode 100644 index 00000000000..c3c7bf27a84 --- /dev/null +++ b/crates/cli/src/container/publish/tests.rs @@ -0,0 +1,691 @@ +//! These fixtures own numeric-loopback sockets and in-memory credentials. They +//! never load CLI configuration, environment endpoints, or external builders. +pub(crate) mod environment_tests; +use super::*; +use axum::{ + body::Bytes, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + Json, Router, +}; +use client::{ObjectRef, UploadKind, UploadStatus}; +use journal::{Record, UploadRecord}; +use serde_json::json; +use spacetimedb_lib::{ + deployment::{api::*, manifest::*, *}, + Identity, Uuid, +}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +const TOKEN: &str = "Bearer isolated-fixture-credential"; + +pub(crate) fn temporary_directory() -> tempfile::TempDir { + let mut builder = tempfile::Builder::new(); + builder.prefix("publication-test-"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + // The fixture itself must satisfy the production ancestor policy. + // tempfile's default directory mode depends on the process umask. + builder.permissions(std::fs::Permissions::from_mode(0o700)); + builder.tempdir_in("/tmp").unwrap() + } + #[cfg(not(unix))] + builder.tempdir().unwrap() +} + +pub(crate) fn empty_module_artifact() -> ModuleArtifact { + let bytes = system_empty::empty().bytes.as_ref(); + ModuleArtifact { + digest: spacetimedb_oci::sha256(bytes), + size_bytes: bytes.len() as u64, + } +} + +pub(crate) fn publisher() -> Identity { + Identity::from_be_byte_array([42; 32]) +} +pub(crate) fn database() -> Identity { + Identity::from_be_byte_array([43; 32]) +} +#[derive(Default)] +pub(crate) struct Behavior { + pub permission: bool, + pub lose_append: bool, + pub lose_completion: bool, + pub lose_submit_after_commit: bool, + pub lose_submit_before_commit: bool, + pub lose_reservation: bool, + pub stale: bool, + pub bad_receipt: bool, + pub bad_completion: bool, + pub bad_status: bool, + pub malformed_status: bool, + pub wrong_reservation: bool, + pub deny_preflight: bool, + pub deny_upload: bool, + pub deny_status: bool, + pub fail_naming: bool, + pub redirect: Option, + pub prior: Option, + pub selected_schema: Option<(spacetimedb_lib::Hash, Vec)>, + pub wrong_module_identity: bool, + pub module_gets: usize, + pub uploads: BTreeMap)>, + pub submits: Vec>, + pub reservations: Vec, + pub names: usize, + pub preflights: usize, + pub begin_count: usize, + pub status: Option, + pub authenticated: usize, +} +pub(crate) struct Fixture { + pub endpoint: String, + pub state: Arc>, + stop: CancellationToken, + task: tokio::task::JoinHandle<()>, +} +impl Fixture { + pub async fn new() -> Self { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(Behavior { + permission: true, + ..Default::default() + })); + let stop = CancellationToken::new(); + let cancellation = stop.clone(); + let app = Router::new() + .fallback(handler) + .layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024)) + .with_state((state.clone(), endpoint.clone())); + let task = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(cancellation.cancelled_owned()) + .await + .unwrap(); + }); + Self { + endpoint, + state, + stop, + task, + } + } + pub fn client(&self) -> PublisherClient { + PublisherClient::new(&self.endpoint, TOKEN.parse().unwrap()).unwrap() + } + pub fn record(&self, new: bool, name: bool) -> Record { + let bytes = system_empty::empty().bytes.as_ref(); + let envelope = PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: None, + expected_last_operation: None, + module_action: ModuleAction::Set(UserModule { + kind: UserModuleKind::Wasm, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }), + container_action: Default::default(), + }; + let module_artifact = empty_module_artifact(); + let request = PublishRequest { + environment: Default::default(), + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + deployment: envelope.resolve(None, &Default::default()).unwrap(), + envelope, + module_artifact, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: new.then_some(CreationOptions { + parent: None, + organization: None, + num_replicas: None, + enforce_anti_affinity: true, + }), + image_source: None, + }; + let request_json = serde_json::to_string(&request).unwrap(); + Record { + version: 2, + server: self.endpoint.clone(), + artifact_endpoint: self.endpoint.clone(), + publisher: publisher(), + database: (!new).then_some(database()), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name: name.then_some("fixture-name".into()), + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads: vec![UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }], + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + } + } + pub async fn close(self) { + self.stop.cancel(); + self.task.await.unwrap(); + } +} +async fn handler( + State((shared, endpoint)): State<(Arc>, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let mut state = shared.lock().unwrap(); + let path = uri.path(); + if path == "/v1/containers/capabilities" { + assert!(headers.get("authorization").is_none()); + if let Some(location) = &state.redirect { + return (StatusCode::TEMPORARY_REDIRECT, [("location", location.clone())]).into_response(); + } + return Json(PublicationCapabilities { + version: 1, + enabled: true, + artifact_endpoint: Some(endpoint), + }) + .into_response(); + } + if headers.get("authorization").and_then(|v| v.to_str().ok()) != Some(TOKEN) { + return StatusCode::UNAUTHORIZED.into_response(); + } + state.authenticated += 1; + if path == "/v1/containers/publish-permission" { + return Json(PublishPermission { + identity: publisher(), + can_publish: state.permission, + source_revision: None, + }) + .into_response(); + } + if path == "/v1/containers/reservations" { + let request: ReserveDatabaseRequest = serde_json::from_slice(&body).unwrap(); + if let Some(prior) = state.reservations.first() { + assert_eq!(prior, &request); + } + state.reservations.push(request.clone()); + if std::mem::take(&mut state.lose_reservation) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(DatabaseReservation { + database_identity: database(), + operation_id: if state.wrong_reservation { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) + } else { + request.operation_id + }, + expires_at: "fixture-only".into(), + staging_open: true, + artifact_endpoint: endpoint, + }) + .into_response(); + } + if path.ends_with("/pre_publish") { + state.preflights += 1; + if state.deny_preflight { + return StatusCode::FORBIDDEN.into_response(); + } + assert_eq!(body.as_ref(), system_empty::empty().bytes.as_ref()); + return Json(spacetimedb_client_api_messages::name::PrePublishResult::AutoMigrate( + spacetimedb_client_api_messages::name::PrePublishAutoMigrateResult { + migrate_plan: "fixture migration".into(), + break_clients: false, + token: spacetimedb_lib::Hash::ZERO, + major_version_upgrade: false, + }, + )) + .into_response(); + } + if path.ends_with("/names") { + state.names += 1; + if state.fail_naming { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(spacetimedb_client_api_messages::name::SetDomainsResult::Success).into_response(); + } + if path.contains("/deployment/operations/") { + if state.deny_status { + return StatusCode::FORBIDDEN.into_response(); + } + return state + .status + .clone() + .map(|status| Json(status).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + if path.ends_with("/schema") && method == Method::GET { + state.module_gets += 1; + assert_eq!(uri.query(), Some("version=10")); + let Some((hash, bytes)) = state.selected_schema.clone() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let identity = if state.wrong_module_identity { + publisher() + } else { + database() + }; + return ( + [ + ("content-type", "application/json".to_owned()), + ("x-spacetimedb-module-hash", hash.to_string()), + ("x-spacetimedb-database-identity", identity.to_hex().to_string()), + ], + bytes, + ) + .into_response(); + } + if path.ends_with("/deployment") { + if method == Method::GET { + return state + .prior + .clone() + .map(|p| Json(p).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + assert_eq!(method, Method::PUT); + state.submits.push(body.to_vec()); + if let Some(status) = &state.status { + assert_eq!(state.submits.first().unwrap().as_slice(), body.as_ref()); + return Json(status.clone()).into_response(); + } + if state.stale { + return (StatusCode::CONFLICT, "private-server-error-must-not-be-printed").into_response(); + } + if std::mem::take(&mut state.lose_submit_before_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let request: PublishRequest = serde_json::from_slice(&body).unwrap(); + request.manifest.validate(&Default::default()).unwrap(); + let status = PublicationStatus { + database_identity: database(), + operation_id: request.manifest.current().envelope.operation_id, + phase: PublicationPhase::Complete, + expected_revision: request.manifest.current().envelope.expected_revision, + expected_last_operation: request.manifest.current().envelope.expected_last_operation, + publication_epoch: 1, + proposed_revision: if state.bad_status { + spacetimedb_lib::Hash::ZERO + } else { + request.manifest.current().deployment.revision().unwrap() + }, + error: None, + }; + state.status = Some(status.clone()); + if state.malformed_status { + let mut value = serde_json::to_value(status).unwrap(); + value["phase"] = json!("secret-body-sentinel"); + return Json(value).into_response(); + } + if std::mem::take(&mut state.lose_submit_after_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(status).into_response(); + } + if path.ends_with("/uploads") { + state.begin_count += 1; + if state.deny_upload { + return StatusCode::FORBIDDEN.into_response(); + } + let request: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + headers["x-spacetimedb-artifact-kind"], + request["kind"].as_str().unwrap() + ); + let object: ObjectRef = serde_json::from_value(request["object"].clone()).unwrap(); + let status = UploadStatus { + id: uuid::Uuid::new_v4(), + object, + offset: 0, + expires_at: u64::MAX, + complete: false, + }; + state.uploads.insert(status.id, (status.clone(), vec![])); + return Json(status).into_response(); + } + if let Some((_, tail)) = path.split_once("/uploads/") { + let id: uuid::Uuid = tail.split('/').next().unwrap().parse().unwrap(); + let Some((status, bytes)) = state.uploads.get_mut(&id) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if method == Method::PATCH { + assert!(body.len() <= client::UPLOAD_CHUNK_BYTES); + let offset: u64 = uri.query().unwrap().strip_prefix("offset=").unwrap().parse().unwrap(); + assert_eq!(offset, status.offset); + bytes.extend_from_slice(&body); + status.offset += body.len() as u64; + } else if path.ends_with("/complete") { + assert_eq!(bytes.len() as u64, status.object.size); + assert_eq!(spacetimedb_oci::sha256(bytes), status.object.digest); + status.complete = true; + } + let mut response = status.clone(); + if path.ends_with("/complete") { + if std::mem::take(&mut state.lose_completion) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let mut object = response.object; + if state.bad_completion { + object.size += 1; + } + return Json(object).into_response(); + } + if method == Method::PATCH && std::mem::take(&mut state.lose_append) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if state.bad_receipt { + response.object.size += 1; + } + return Json(response).into_response(); + } + panic!("unexpected fixture request: {method} {uri}"); +} +async fn run_now(client: &PublisherClient, journal: &mut Journal) -> Result { + run(client, journal, None, Duration::ZERO, CancellationToken::new()).await +} +fn journal(base: &Path, record: Record) -> Journal { + Journal::create(base, record, None, Some(system_empty::empty().bytes.as_ref())).unwrap() +} +use std::path::Path; + +#[tokio::test] +async fn lost_append_and_committed_response_resume_without_reupload_or_current_permission() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_append = true; + s.lose_submit_after_commit = true; + } + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.submitted); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + fixture.state.lock().unwrap().permission = false; + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.into_bytes()]); + assert_eq!(state.begin_count, 1); + } + drop(journal); + fixture.close().await; +} +#[tokio::test] +async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_identity() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_reservation = true; + s.lose_submit_before_commit = true; + } + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(true, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.database.is_none()); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(journal.record.database, Some(database())); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.reservations.len(), 2); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} +#[tokio::test] +async fn stale_revision_preserves_operation_and_redacts_response_without_fallback() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().stale = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(!format!("{error:#}").contains("private-server-error")); + assert_eq!(journal.record.request_json, exact); + assert!(journal.record.submitted); + assert_eq!(fixture.state.lock().unwrap().submits.len(), 1); + fixture.close().await; +} +#[tokio::test] +async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { + for reservation in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.wrong_reservation = reservation; + s.bad_receipt = !reservation; + } + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(reservation, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(!journal.record.submitted); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } +} +#[tokio::test] +async fn completion_descriptor_must_match_before_admission() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().bad_completion = true; + let directory = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(error.to_string().contains("completion descriptor changed")); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + assert!(!journal.record.submitted); + fixture.state.lock().unwrap().bad_completion = false; + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let status = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(status.complete); + assert_eq!(status.offset, status.object.size); + fixture.close().await; +} + +#[tokio::test] +async fn lost_completion_response_observes_the_same_complete_session() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_completion = true; + let directory = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let session = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(session.complete); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.begin_count, 1); + assert_eq!(state.uploads.len(), 1); + assert!(state.uploads[&session.id].0.complete); + } + fixture.close().await; +} + +#[tokio::test] +async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { + for denied in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.deny_upload = denied; + s.bad_status = !denied; + } + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.status.is_none()); + assert_eq!(fixture.state.lock().unwrap().submits.len(), usize::from(!denied)); + fixture.close().await; + } +} +#[tokio::test] +async fn naming_failure_reports_active_identity_without_republishing_or_overwriting_later_names() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().fail_naming = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(true, true)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.names, 1); + assert_eq!(state.submits.len(), 1); + } + fixture.close().await; +} +#[tokio::test] +async fn redirect_and_cross_origin_never_forward_credentials_without_exact_approval() { + let first = Fixture::new().await; + let foreign = Fixture::new().await; + first.state.lock().unwrap().redirect = Some(format!("{}v1/containers/publish-permission", foreign.endpoint)); + assert!(first.client().capabilities().await.is_err()); + assert!(first.client().artifact_endpoint(&foreign.endpoint, None).is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&first.endpoint)) + .is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&foreign.endpoint)) + .is_ok()); + assert_eq!(foreign.state.lock().unwrap().authenticated, 0); + first.close().await; + foreign.close().await; +} +#[tokio::test] +async fn journal_locks_and_reverifies_local_bytes_without_persisting_credentials() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let path = journal.directory().to_owned(); + assert!(Journal::open(&path).is_err()); + let bytes = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!bytes.contains("isolated-fixture-credential")); + std::fs::write( + path.join("module.blob"), + vec![0; system_empty::empty().bytes.as_ref().len()], + ) + .unwrap(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(fixture.state.lock().unwrap().begin_count, 0); + drop(journal); + let mut value: serde_json::Value = serde_json::from_str(&bytes).unwrap(); + value["request_json"] = json!("{}"); + std::fs::write(path.join("publication.json"), serde_json::to_vec(&value).unwrap()).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} +#[test] +fn endpoints_and_receipts_are_bounded_and_unambiguous() { + for endpoint in [ + "Maincloud", + "http://10.0.0.1", + "https://user:password@example.invalid", + "https://example.invalid/?token=secret", + ] { + assert!(client::endpoint(endpoint).is_err()); + } + for endpoint in ["http://localhost:3000", "http://[::1]:3000"] { + assert!(client::endpoint(endpoint).is_ok()); + } + let object = ObjectRef { + digest: empty_module_artifact().digest, + size: 1, + }; + let status = UploadStatus { + id: uuid::Uuid::now_v7(), + object, + offset: 0, + expires_at: 0, + complete: false, + }; + assert!(status.validate(object, None).is_err()); +} + +#[tokio::test] +async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutates_server() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let base = temporary.path().join("new").join("nested").join("state"); + let record = fixture.record(false, false); + let id = record.request().unwrap().manifest.current().envelope.operation_id; + // A missing promised module fails the artifact flush before publication.json + // can make this operation available for reservation or admission. + assert!(Journal::create(&base, record, None, None).is_err()); + assert!(!base.join(id.to_string()).exists()); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn revoked_status_access_replays_original_put_without_missing_local_artifacts_or_new_uploads() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.deny_status = true; + state.deny_upload = true; + } + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} diff --git a/crates/cli/src/container/publish/tests/environment_tests.rs b/crates/cli/src/container/publish/tests/environment_tests.rs new file mode 100644 index 00000000000..10ad97b24a6 --- /dev/null +++ b/crates/cli/src/container/publish/tests/environment_tests.rs @@ -0,0 +1,207 @@ +use super::*; + +fn set_environment(record: &mut Record, values: BTreeMap) { + let mut request = record.request().unwrap(); + request.environment = values; + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); +} + +#[tokio::test] +async fn complete_large_environment_is_private_and_exact_after_lost_reply_and_revocation() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut record = fixture.record(false, false); + set_environment( + &mut record, + (0..128) + .map(|n| (format!("KEY_{n}"), "secret-body-sentinel".repeat(256))) + .collect(), + ); + assert!(record.request_json.len() > 256 * 1024); + let exact = record.request_json.clone(); + let mut retained = journal(temporary.path(), record); + let path = retained.directory().to_owned(); + let metadata = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!metadata.contains("secret-body-sentinel") && !metadata.contains("request_json")); + assert_eq!(std::fs::read(path.join("submission.json")).unwrap(), exact.as_bytes()); + assert!(run_now(&fixture.client(), &mut retained).await.is_err()); + drop(retained); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_status = true; + state.permission = false; + state.deny_upload = true; + } + let mut retained = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut retained).await.unwrap(), + Outcome::Complete(_) + )); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + drop(retained); + fixture.close().await; +} + +#[tokio::test] +async fn missing_changed_and_malformed_secret_body_never_becomes_empty_or_leaks_in_errors() { + let fixture = Fixture::new().await; + for replacement in [ + None, + Some(b"{}".as_slice()), + Some(br#"{"environment":{"KEY":"secret-body-sentinel", "KEY":17}}"#.as_slice()), + ] { + let temporary = crate::container::publish::tests::temporary_directory(); + let mut record = fixture.record(false, false); + set_environment(&mut record, BTreeMap::from([("KEY".into(), "original-secret".into())])); + let retained = journal(temporary.path(), record); + let path = retained.directory().to_owned(); + if let Some(bytes) = replacement { + std::fs::write(path.join("submission.json"), bytes).unwrap(); + } else { + std::fs::remove_file(path.join("submission.json")).unwrap(); + } + assert!(retained.submission_bytes().is_err()); + drop(retained); + let error = Journal::open(&path).err().unwrap(); + assert!(!format!("{error:#}").contains("secret-body-sentinel")); + } + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn status_must_match_previous_operation_and_the_first_confirmed_epoch() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut retained = journal(temporary.path(), fixture.record(false, false)); + run_now(&fixture.client(), &mut retained).await.unwrap(); + let request = retained.record.request().unwrap(); + for which in 0..3 { + let mut status = retained.record.status.clone().unwrap(); + match which { + 0 => status.expected_last_operation = Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + 1 => status.publication_epoch = 0, + _ => status.publication_epoch += 1, + } + assert!(retained.record.check_status(&request, &status).is_err()); + } + fixture.close().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn protected_storage_rejects_public_modes_symlinks_and_hardlinks() { + use std::os::unix::fs::{symlink, PermissionsExt}; + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let base = temporary.path().join("new/retained"); + let retained = journal(&base, fixture.record(false, false)); + for parent in [temporary.path().join("new"), base] { + assert_eq!(std::fs::metadata(parent).unwrap().permissions().mode() & 0o777, 0o700); + } + let path = retained.directory().to_owned(); + let body = path.join("submission.json"); + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o700); + for name in ["submission.json", "publication.json", "publication.lock"] { + assert_eq!( + std::fs::metadata(path.join(name)).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + std::fs::set_permissions(&body, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(retained.submission_bytes().is_err()); + std::fs::set_permissions(&body, std::fs::Permissions::from_mode(0o600)).unwrap(); + let link = temporary.path().join("extra-link"); + std::fs::hard_link(&body, &link).unwrap(); + assert!(retained.submission_bytes().is_err()); + std::fs::remove_file(&link).unwrap(); + std::fs::rename(&body, &link).unwrap(); + symlink(&link, &body).unwrap(); + assert!(retained.submission_bytes().is_err()); + drop(retained); + assert!(Journal::open(&path).is_err()); + std::fs::remove_file(&body).unwrap(); + std::fs::rename(&link, &body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} + +pub(crate) fn schema_bytes(environment: spacetimedb_lib::environment::EnvironmentSchema) -> Vec { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + serde_json::to_vec(&spacetimedb_lib::sats::serde::SerdeWrapper(RawModuleDefV10 { + sections: vec![ + RawModuleDefV10Section::Typespace(Default::default()), + RawModuleDefV10Section::Environment(environment.into_declarations()), + ], + })) + .unwrap() +} + +#[tokio::test] +async fn kept_schema_checks_identity_program_hash_and_bounded_v10_metadata() { + let fixture = Fixture::new().await; + let request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + last_operation: Some(request.manifest.current().envelope.operation_id), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: empty_module_artifact().digest, + size_bytes: empty_module_artifact().size_bytes, + }, + }; + let hash = system_empty::empty().descriptor.program_hash; + fixture.state.lock().unwrap().selected_schema = Some((hash, schema_bytes(Default::default()))); + assert_eq!( + fixture.client().selected_environment(&prior).await.unwrap(), + Default::default() + ); + fixture.state.lock().unwrap().wrong_module_identity = true; + assert!(fixture.client().selected_environment(&prior).await.is_err()); + fixture.state.lock().unwrap().wrong_module_identity = false; + for (hash, bytes) in [ + (spacetimedb_lib::Hash::ZERO, schema_bytes(Default::default())), + (hash, b"invalid-private-schema-sentinel".to_vec()), + (hash, vec![0; 16 * 1024 * 1024 + 1]), + ] { + fixture.state.lock().unwrap().selected_schema = Some((hash, bytes)); + let error = fixture.client().selected_environment(&prior).await.unwrap_err(); + assert!(!format!("{error:#}").contains("invalid-private-schema-sentinel")); + } + fixture.close().await; +} + +#[tokio::test] +async fn malformed_server_response_cannot_disclose_echoed_environment_values() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().malformed_status = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut retained = journal(temporary.path(), fixture.record(false, false)); + let error = run_now(&fixture.client(), &mut retained).await.unwrap_err(); + assert!(!format!("{error:#}").contains("secret-body-sentinel")); + assert!(retained.record.submitted && retained.record.status.is_none()); + fixture.close().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn untrusted_writable_parent_is_rejected_before_retaining_secrets() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let base = temporary.path().join("untrusted"); + std::fs::create_dir(&base).unwrap(); + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o777)).unwrap(); + let record = fixture.record(false, false); + assert!(Journal::create(&base, record, None, Some(system_empty::empty().bytes.as_ref())).is_err()); + assert_eq!(std::fs::read_dir(&base).unwrap().count(), 0); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs new file mode 100644 index 00000000000..9dabee6c96d --- /dev/null +++ b/crates/cli/src/container/tests.rs @@ -0,0 +1,774 @@ +use super::*; +use crate::spacetime_config::SpacetimeConfig; +use serde_json::json; +use std::{collections::BTreeMap, sync::Mutex}; + +static TEST_PREPARATIONS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "amd64".into(), + } +} +pub(crate) fn declaration(image: serde_json::Value) -> ContainerConfig { + serde_json::from_value(json!({"image":image,"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}})).unwrap() +} +fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { + let digest = spacetimedb_oci::sha256(bytes); + fs::create_dir_all(layout.join("blobs/sha256")).unwrap(); + fs::write( + layout + .join("blobs/sha256") + .join(digest.to_string().strip_prefix("sha256:").unwrap()), + bytes, + ) + .unwrap(); + Descriptor { + media_type: media.into(), + digest, + size: bytes.len() as u64, + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +pub(crate) fn fixture(layout: &Path) -> Descriptor { + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, "app.js", &b"code"[..]).unwrap(); + let layer = archive.into_inner().unwrap(); + let layer = blob(layout, &layer, "application/vnd.oci.image.layer.v1.tar"); + let config = blob(layout, &serde_json::to_vec(&json!({"architecture":"amd64","os":"linux","config":{"Entrypoint":["/usr/bin/env"],"Cmd":["node","app.js"],"User":"1000:1000","WorkingDir":"/app","Env":["BAKED=private-image-value"]},"rootfs":{"type":"layers","diff_ids":[layer.digest]}})).unwrap(), spacetimedb_oci::OCI_CONFIG); + let manifest = blob( + layout, + &serde_json::to_vec( + &json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_MANIFEST,"config":config,"layers":[layer]}), + ) + .unwrap(), + spacetimedb_oci::OCI_MANIFEST, + ); + fs::write(layout.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#).unwrap(); + fs::write( + layout.join("index.json"), + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_INDEX,"manifests":[manifest]})) + .unwrap(), + ) + .unwrap(); + manifest +} +fn copy_layout(source: &Path, output: &Path) { + fs::create_dir_all(output.join("blobs/sha256")).unwrap(); + for name in ["oci-layout", "index.json"] { + fs::copy(source.join(name), output.join(name)).unwrap(); + } + for file in fs::read_dir(source.join("blobs/sha256")).unwrap() { + let file = file.unwrap(); + fs::copy(file.path(), output.join("blobs/sha256").join(file.file_name())).unwrap(); + } +} +type RecordedCall = (String, Vec, Vec); + +struct FakeRunner { + layout: PathBuf, + calls: Mutex>, + fail_detection: bool, + version: &'static str, +} +impl FakeRunner { + fn new(layout: &Path) -> Self { + Self { + layout: layout.into(), + calls: Mutex::new(vec![]), + fail_detection: false, + version: "railpack 0.35.0\n", + } + } +} +impl Runner for FakeRunner { + async fn run(&self, invocation: Invocation) -> Result { + self.calls.lock().unwrap().push(( + invocation.label.into(), + invocation.args.clone(), + invocation.env.iter().map(|(name, _)| name.clone()).collect(), + )); + match invocation.label { + "Railpack version check" => { + return Ok(process::Output { + stdout: self.version.as_bytes().to_vec(), + }) + } + "Railpack detection" => { + ensure!(!self.fail_detection, "unsupported source detection"); + fs::write(invocation.workspace.path().join("railpack-plan.json"), b"{}")?; + } + "Skopeo image import" => copy_layout(&self.layout, &invocation.workspace.path().join("input")), + "BuildKit OCI build" => { + let file = fs::File::create(invocation.workspace.path().join("image.tar"))?; + let mut tar = tar::Builder::new(file); + for name in ["oci-layout", "index.json"] { + tar.append_path_with_name(self.layout.join(name), name)?; + } + for blob in fs::read_dir(self.layout.join("blobs/sha256"))? { + let blob = blob?; + tar.append_path_with_name(blob.path(), Path::new("blobs/sha256").join(blob.file_name()))?; + } + tar.finish()?; + } + _ => anyhow::bail!("unexpected fake image tool"), + } + Ok(process::Output { stdout: vec![] }) + } +} + +#[test] +fn image_source_is_exclusive_and_builder_defaults_are_strict() { + for image in [ + json!({}), + json!({"build":{},"oci_ref":"example/image"}), + json!({"build":{"builder":"unknown"}}), + json!({"build":{"builder":"railpack","dockerfile":"Dockerfile"}}), + ] { + let mut config = serde_json::to_value(declaration(json!({"oci_ref":"example/image"}))).unwrap(); + config["image"] = image; + assert!(serde_json::from_value::(config).is_err()); + } + assert!(matches!( + declaration(json!({"build":{}})).image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Dockerfile { .. } + }) + )); +} + +#[test] +fn container_declarations_never_inherit_through_children_or_overrides() { + let a = serde_json::to_value(declaration(json!({"build":{}}))).unwrap(); + let b = serde_json::to_value(declaration(json!({"oci_ref":"example/child"}))).unwrap(); + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"root","module-path":"shared","container":a,"children":[{"database":"plain","children":[{"database":"grandchild"}]},{"database":"own","container":b,"children":[{"database":"own-grandchild"}]}]})).unwrap(); + let targets = config.collect_all_targets_with_inheritance(); + let declarations: BTreeMap<_, _> = targets + .iter() + .map(|target| (target.fields["database"].as_str().unwrap(), target.container.is_some())) + .collect(); + assert_eq!( + declarations, + BTreeMap::from([ + ("root", true), + ("plain", false), + ("grandchild", false), + ("own", true), + ("own-grandchild", false) + ]) + ); + assert!(targets.iter().all(|target| !target.fields.contains_key("container"))); + assert!(crate::subcommands::container::select(&config, Some("plain")).is_err()); + assert!(crate::subcommands::container::select(&config, Some("not-a-local-target")).is_err()); + assert!(crate::subcommands::container::select(&config, None).is_err()); + assert!(crate::subcommands::container::select(&config, Some("own")).is_ok()); +} + +#[tokio::test] +async fn local_prebuilt_output_is_owned_verified_and_keeps_image_defaults() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("prebuilt image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let token = CancellationToken::new(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:prebuilt image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token.clone(), + ) + .await + .unwrap(); + assert!(!token.is_cancelled()); + assert!(runner.calls.lock().unwrap().is_empty()); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + assert_eq!(prepared.metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(prepared.metadata.container.user, "1000:1000"); + assert_eq!(prepared.metadata.container.working_directory, "/app"); + assert!(!serde_json::to_string(&prepared.metadata) + .unwrap() + .contains("private-image-value")); + assert_eq!(prepared.metadata.objects.len(), 3); + let temporary = prepared.layout(); + let output = root.path().join("reusable"); + prepared.persist(&output).unwrap(); + assert!(!temporary.exists()); + assert!(output.join("prepared.json").is_file()); + let reread = prepare_container( + &declaration(json!({"oci_ref":"oci:reusable"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token, + ) + .await + .unwrap(); + assert_eq!(reread.metadata.manifest.digest, manifest.digest); + let temporary = reread.layout(); + drop(reread); + assert!(!temporary.exists()); +} + +#[tokio::test] +async fn command_override_replaces_image_argv_and_invalid_bytes_never_persist() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let mut config = declaration(json!({"oci_ref":"oci:image"})); + config.command = Some(vec!["/bin/custom".into()]); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.container.argv, ["/bin/custom"]); + drop(prepared); + fs::write( + input + .join("blobs/sha256") + .join(manifest.digest.to_string().strip_prefix("sha256:").unwrap()), + b"tampered", + ) + .unwrap(); + assert!(prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(fs::read_dir(root.path()).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".spacetime-image-"))); +} + +#[tokio::test] +async fn dockerfile_and_railpack_have_explicit_local_endpoint_and_separate_secrets() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("fixture"); + fixture(&input); + let context = root.path().join("source with spaces"); + fs::create_dir(&context).unwrap(); + fs::write(context.join("Dockerfile"), b"FROM scratch").unwrap(); + let secret = root.path().join("secret file"); + fs::write(&secret, b"sensitive-build-value").unwrap(); + for builder in ["dockerfile", "railpack"] { + let runner = FakeRunner::new(&input); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + secrets: vec![BuildSecret { + name: "BUILD_KEY".into(), + file: secret.clone(), + }], + ..Default::default() + }; + let config = declaration(json!({"build":{"builder":builder,"context":"source with spaces"}})); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + let calls = runner.calls.lock().unwrap(); + let (_, argv, environment) = calls + .iter() + .find(|(label, _, _)| label == "BuildKit OCI build") + .unwrap(); + assert_eq!(&argv[..2], ["--addr", "unix:///disposable/fake-buildkit.sock"]); + assert!(argv.contains(&format!("context={}", context.canonicalize().unwrap().display()).into())); + assert!(argv.contains(&"--no-cache".into())); + assert!(environment.contains(&"DOCKER_CONFIG".into())); + assert!(!format!("{argv:?}").contains("sensitive-build-value")); + assert!(!serde_json::to_string(&prepared.metadata).unwrap().contains("BUILD_KEY")); + if builder == "railpack" { + assert!(argv.contains(&format!("source={RAILPACK_FRONTEND}").into())); + let (_, args, _) = calls + .iter() + .find(|(label, _, _)| label == "Railpack detection") + .unwrap(); + assert!(args.contains(&"BUILD_KEY=".into())); + } else { + assert_eq!(calls.len(), 1); + } + } +} + +#[tokio::test] +async fn unsupported_railpack_and_remote_builder_never_fall_back() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let config = declaration(json!({"build":{"builder":"railpack"}})); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + ..Default::default() + }; + let mut runner = FakeRunner::new(&input); + runner.fail_detection = true; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 2); + runner.calls.lock().unwrap().clear(); + runner.version = "railpack 99.0.0"; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 1); + runner.calls.lock().unwrap().clear(); + let tools = BuildTools { + buildkit_host: Some("tcp://untrusted:1234".into()), + ..Default::default() + }; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(runner.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn registry_import_uses_explicit_anonymous_auth_and_records_selected_digest() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:tag"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + let calls = runner.calls.lock().unwrap(); + assert_eq!(calls.len(), 1); + let args = &calls[0].1; + assert!(args.contains(&"--preserve-digests".into())); + assert!(args.contains(&"docker://example.invalid/team/image:tag".into())); + let auth = PathBuf::from(&args[args.iter().position(|arg| arg == "--authfile").unwrap() + 1]); + assert_eq!(fs::read(auth).unwrap(), br#"{"auths":{}}"#); +} + +#[cfg(unix)] +#[tokio::test] +async fn registry_credentials_are_private_during_preparation_under_an_existing_readable_base() { + use std::os::unix::fs::PermissionsExt; + let _serial = TEST_PREPARATIONS.lock().await; + let root = crate::container::publish::tests::temporary_directory(); + let input = root.path().join("image"); + fixture(&input); + let base = root.path().join("existing"); + fs::create_dir(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let credential = root.path().join("registry.json"); + const AUTH: &[u8] = br#"{"auths":{"example.invalid":{"auth":"fixture-registry-secret"}}}"#; + fs::write(&credential, AUTH).unwrap(); + fs::set_permissions(&credential, fs::Permissions::from_mode(0o600)).unwrap(); + struct InspectingRunner { + inner: FakeRunner, + workspace: Mutex>, + } + impl Runner for InspectingRunner { + async fn run(&self, invocation: Invocation) -> Result { + let workspace = invocation.workspace.path(); + assert_eq!(fs::metadata(workspace).unwrap().permissions().mode() & 0o777, 0o700); + assert_eq!(fs::read(workspace.join("auth/config.json")).unwrap(), AUTH); + *self.workspace.lock().unwrap() = Some(workspace.to_path_buf()); + self.inner.run(invocation).await + } + } + let runner = InspectingRunner { + inner: FakeRunner::new(&input), + workspace: Mutex::new(None), + }; + let tools = BuildTools { + registry_auth_file: Some(credential), + ..Default::default() + }; + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:fixture"})), + root.path(), + platform(), + &tools, + &base, + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(fs::metadata(&base).unwrap().permissions().mode() & 0o777, 0o755); + let workspace = runner.workspace.lock().unwrap().clone().unwrap(); + assert!(workspace.exists()); + drop(prepared); + assert!(!workspace.exists()); +} + +#[test] +fn archive_rejects_links_and_cancelled_reads() { + let root = tempfile::tempdir().unwrap(); + for kind in [tar::EntryType::Symlink, tar::EntryType::Link] { + let path = root.path().join("input.tar"); + let mut tar = tar::Builder::new(fs::File::create(&path).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_size(0); + header.set_link_name("/outside").unwrap(); + header.set_cksum(); + tar.append_data(&mut header, "index.json", std::io::empty()).unwrap(); + tar.finish().unwrap(); + let output = tempfile::tempdir().unwrap(); + assert!(oci::extract_archive( + &path, + output.path(), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(1) + ) + .is_err()); + } + let token = CancellationToken::new(); + token.cancel(); + assert!(oci::check(&token, Instant::now() + Duration::from_secs(1)).is_err()); +} + +#[test] +fn container_build_command_requires_explicit_platform_and_output() { + crate::subcommands::container::cli().debug_assert(); + assert!(crate::subcommands::container::cli() + .try_get_matches_from(["container", "build"]) + .is_err()); + assert!(crate::subcommands::container::cli() + .try_get_matches_from([ + "container", + "build", + "local-target", + "--platform", + "linux/amd64", + "--out-dir", + "output" + ]) + .is_ok()); +} + +#[test] +fn publish_preserves_container_target_metadata_without_affecting_plain_children() { + use crate::subcommands::publish::{build_publish_schema, get_filtered_publish_configs}; + let config: SpacetimeConfig = serde_json::from_value(json!({ + "database":"container-db", "container":declaration(json!({"build":{}})), + "children":[{"database":"ordinary-db"}] + })) + .unwrap(); + let command = crate::subcommands::publish::cli(); + let schema = build_publish_schema(&command).unwrap(); + for selected in ["container-db", "*"] { + let args = command.clone().try_get_matches_from(["publish", selected]).unwrap(); + let targets = get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert!(targets[0].container().is_some()); + assert!(targets.iter().skip(1).all(|target| target.container().is_none())); + } + let args = command + .clone() + .try_get_matches_from(["publish", "ordinary-db"]) + .unwrap(); + assert_eq!( + get_filtered_publish_configs(&config, &command, &schema, &args) + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn local_index_defaults_media_type_but_never_overwrites_existing_output() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let mut index: serde_json::Value = serde_json::from_slice(&fs::read(input.join("index.json")).unwrap()).unwrap(); + index.as_object_mut().unwrap().remove("mediaType"); + fs::write(input.join("index.json"), serde_json::to_vec(&index).unwrap()).unwrap(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &FakeRunner::new(&input), + CancellationToken::new(), + ) + .await + .unwrap(); + let output = root.path().join("existing-empty-output"); + fs::create_dir(&output).unwrap(); + assert!(prepared.persist(&output).is_err()); + assert!(fs::read_dir(&output).unwrap().next().is_none()); +} + +#[test] +fn wrong_platform_and_duplicate_archive_object_are_rejected() { + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let wrong = ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }; + assert!(oci::verify_layout( + &input, + &root.path().join("output"), + &wrong, + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); + let archive = root.path().join("duplicate.tar"); + let mut tar = tar::Builder::new(fs::File::create(&archive).unwrap()); + for _ in 0..2 { + tar.append_path_with_name(input.join("index.json"), "index.json") + .unwrap(); + } + tar.finish().unwrap(); + assert!(oci::extract_archive( + &archive, + &root.path().join("duplicate"), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_before_workspace_release() { + use process::LocalRunner; + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + // These scripts exercise only owned local subprocesses. No Docker, server, + // network, saved configuration, or user credentials are involved. + for mode in [ + "exit", + "exit_without_descendants", + "cancel", + "drop", + "overflow", + "timeout", + ] { + let workspace = Arc::new( + tempfile::Builder::new() + .prefix("fake-tool-") + .tempdir_in(root.path()) + .unwrap(), + ); + let path = workspace.path().to_path_buf(); + let pid_file = root.path().join(format!("{mode}.pid")); + let script = match mode { + "exit" => "echo $$ > \"$1\"; sleep 60 & exit 0", + "exit_without_descendants" => "echo $$ > \"$1\"; exit 0", + "overflow" => "echo $$ > \"$1\"; yes x", + _ => "echo $$ > \"$1\"; sleep 60 & wait", + }; + let token = CancellationToken::new(); + let task = tokio::spawn(LocalRunner.run(Invocation { + tool: "/bin/sh".into(), + label: "owned fake builder", + args: vec![ + "-c".into(), + script.into(), + "fake-builder".into(), + pid_file.clone().into_os_string(), + ], + env: vec![], + cwd: root.path().into(), + workspace, + timeout: if mode == "timeout" { + Duration::from_millis(250) + } else { + Duration::from_secs(4) + }, + cancel: token.clone(), + })); + tokio::time::timeout(Duration::from_secs(2), async { + while !pid_file.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let pid = fs::read_to_string(&pid_file).unwrap().trim().parse::().unwrap(); + let pid = rustix::process::Pid::from_raw(pid).unwrap(); + if mode == "cancel" { + token.cancel(); + } + if mode == "drop" { + task.abort(); + let _ = task.await; + } else { + let result = tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert_eq!(result.is_ok(), matches!(mode, "exit" | "exit_without_descendants")); + } + tokio::time::timeout(Duration::from_secs(3), async { + while path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!( + matches!( + rustix::process::waitpid(Some(pid), rustix::process::WaitOptions::NOHANG), + Err(rustix::io::Errno::CHILD) + ), + "builder leader must already be reaped" + ); + } +} + +#[test] +fn configuration_overlay_replaces_only_the_selected_container_declaration() { + let root = tempfile::tempdir().unwrap(); + let parent = declaration(json!({"build":{}})); + let child = declaration(json!({"oci_ref":"example.invalid/child:v1"})); + fs::write(root.path().join("spacetime.json"),serde_json::to_vec(&json!({"database":"parent","container":parent,"children":[{"database":"plain"},{"database":"own","container":child}]})).unwrap()).unwrap(); + let replacement = declaration(json!({"build":{"builder":"railpack"}})); + fs::write( + root.path().join("spacetime.dev.json"), + serde_json::to_vec(&json!({"container":replacement})).unwrap(), + ) + .unwrap(); + let loaded = crate::spacetime_config::find_and_load_with_env_from(Some("dev"), root.path().into()) + .unwrap() + .unwrap(); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("parent")) + .unwrap() + .image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Railpack { .. } + }) + )); + assert!(crate::subcommands::container::select(&loaded.config, Some("plain")).is_err()); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("own")) + .unwrap() + .image, + ImageSource::Prebuilt(_) + )); +} + +#[tokio::test] +async fn build_command_executes_local_import_with_only_project_configuration() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + fixture(&root.path().join("input")); + fs::write( + root.path().join("spacetime.json"), + serde_json::to_vec(&json!({ + "database":"local-selection", "server":"https://must-not-be-contacted.invalid", + "container":declaration(json!({"oci_ref":"oci:input"})) + })) + .unwrap(), + ) + .unwrap(); + let output = root.path().join("output"); + let args = crate::subcommands::container::cli() + .try_get_matches_from([ + OsString::from("container"), + "build".into(), + "local-selection".into(), + "--project-path".into(), + root.path().into(), + "--out-dir".into(), + output.clone().into_os_string(), + "--platform".into(), + "linux/amd64".into(), + ]) + .unwrap(); + crate::exec_local_subcommand("container", &args).await.unwrap().unwrap(); + let metadata: PreparedMetadata = serde_json::from_slice(&fs::read(output.join("prepared.json")).unwrap()).unwrap(); + assert_eq!(metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(metadata.objects.len(), 3); +} + +#[test] +fn omitted_image_working_directory_normalizes_to_linux_root() { + let declaration = declaration(json!({"oci_ref":"example.invalid/no-working-dir"})); + let image = spacetimedb_oci::ContainerConfig { + cmd: Some(vec!["/app".into()]), + ..Default::default() + }; + let normalized = declaration + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .unwrap(); + assert_eq!(normalized.working_directory, "/"); + let mut explicit = declaration; + explicit.working_directory = Some(String::new()); + assert!(explicit + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .is_err()); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 5357e8d067b..053cf20b339 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; mod common_args; mod config; +pub mod container; pub(crate) mod detect; mod edit_distance; mod errors; @@ -38,6 +39,7 @@ pub fn get_subcommands() -> Vec { logout::cli(), init::cli(), build::cli(), + subcommands::container::cli(), server::cli(), subscribe::cli(), start::cli(), @@ -47,6 +49,23 @@ pub fn get_subcommands() -> Vec { ] } +/// Dispatch commands that need only project files before opening saved CLI +/// server settings or credentials. Future container network commands use the +/// ordinary authenticated dispatcher below. +pub async fn exec_local_subcommand(cmd: &str, args: &ArgMatches) -> Option> { + if cmd == "container" + && let Some(("build", args)) = args.subcommand() + { + Some( + subcommands::container::exec_build(args) + .await + .map(|()| ExitCode::SUCCESS), + ) + } else { + None + } +} + pub async fn exec_subcommand( config: Config, paths: &SpacetimePaths, @@ -69,6 +88,7 @@ pub async fn exec_subcommand( "list" => list::exec(config, args).await, "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), + "container" => subcommands::container::exec(config, args).await, "server" => server::exec(config, paths, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d6f58757eea..e5bf606175f 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -32,6 +32,10 @@ async fn main() -> anyhow::Result { let matches = get_command().get_matches(); let (cmd, subcommand_args) = matches.subcommand().unwrap(); + if let Some(result) = exec_local_subcommand(cmd, subcommand_args).await { + return result; + } + let root_dir = matches.get_one::("root_dir"); let paths = match root_dir { Some(dir) => SpacetimePaths::from_root_dir(dir), @@ -135,3 +139,14 @@ Commands: "#, ) } + +#[cfg(test)] +mod tests { + #[test] + fn managed_publish_help_passes_entrypoint_validation() { + let help = super::get_command() + .try_get_matches_from(["spacetime", "publish", "--help"]) + .unwrap_err(); + assert_eq!(help.kind(), clap::error::ErrorKind::DisplayHelp); + } +} diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index 32d3030082a..bf72e54c841 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -110,6 +110,9 @@ pub enum CommandConfigError { #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "kebab-case")] pub struct SpacetimeConfig { + /// Container declaration belongs only to this database, never its children. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, /// Configuration for the dev command. Root-level only, not inherited. #[serde(skip_serializing_if = "Option::is_none")] pub dev: Option, @@ -146,6 +149,7 @@ pub struct DevConfig { /// Contains all fields needed for both publish and generate operations. #[derive(Debug, Clone)] pub struct FlatTarget { + pub container: Option, /// All entity-level fields (database, module-path, server, etc.) pub fields: HashMap, /// Name of the config file from which this target's `database` value was merged. @@ -211,6 +215,7 @@ impl SpacetimeConfig { let effective_generate = self.generate.clone(); let target = FlatTarget { + container: self.container.clone(), fields: fields.clone(), source_config: self.source_config.clone(), generate: effective_generate, @@ -263,6 +268,8 @@ pub struct CommandConfig<'a> { config_values: HashMap, /// CLI arguments matches: &'a ArgMatches, + /// A declaration belongs to this exact target and never inherits. + container: Option, } /// Schema that defines the contract between CLI arguments and config file keys. @@ -738,9 +745,23 @@ impl<'a> CommandConfig<'a> { schema, config_values: normalized_values, matches, + container: None, }) } + pub fn with_container(mut self, container: Option) -> Self { + self.container = container; + self + } + + pub fn container(&self) -> Option<&crate::container::config::ContainerConfig> { + self.container.as_ref() + } + + pub(crate) fn matches(&self) -> &ArgMatches { + self.matches + } + /// Get a single value from the config as a specific type. /// First checks clap args (via schema), then falls back to config values. /// diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs new file mode 100644 index 00000000000..8e5d2a3b5d9 --- /dev/null +++ b/crates/cli/src/subcommands/container.rs @@ -0,0 +1,193 @@ +//! Container build and operation commands. Local builds do not read saved +//! server credentials; network commands use the explicitly selected server. +#[path = "container/exec.rs"] +mod execute; +mod logs; +mod operations; +mod url; + +use crate::{ + container::{config::ContainerConfig, prepare_container, process::LocalRunner, BuildSecret, BuildTools}, + spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}, +}; +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use std::path::{Path, PathBuf}; +use tokio_util::sync::CancellationToken; + +pub fn cli() -> Command { + let command = Command::new("container") + .about("Build and manage a database's container") + .subcommand_required(true) + .subcommand(url::cli()) + .subcommand(execute::cli()) + .subcommand(logs::cli()) + .subcommand( + Command::new("build") + .about("Prepare verified OCI artifacts locally without publishing") + .arg(Arg::new("database").help("Database target in local spacetime.json; no server lookup")) + .arg( + Arg::new("project_path") + .long("project-path") + .default_value(".") + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory in which to find spacetime.json"), + ) + .arg( + Arg::new("out_dir") + .long("out-dir") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("New directory for verified OCI artifacts and prepared.json"), + ) + .arg( + Arg::new("platform") + .long("platform") + .required(true) + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Target Linux platform, independent of this computer's architecture"), + ) + .arg(Arg::new("env").long("env").help("Local configuration overlay name")) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket, required for source builds"), + ) + .arg( + Arg::new("buildctl") + .long("buildctl") + .default_value("buildctl") + .value_parser(clap::value_parser!(PathBuf)) + .help("BuildKit client executable"), + ) + .arg( + Arg::new("railpack") + .long("railpack") + .default_value("railpack") + .value_parser(clap::value_parser!(PathBuf)) + .help("Pinned Railpack executable for explicitly selected Railpack builds"), + ) + .arg( + Arg::new("skopeo") + .long("skopeo") + .default_value("skopeo") + .value_parser(clap::value_parser!(PathBuf)) + .help("Skopeo executable for prebuilt registry images"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; omitted means anonymous, never saved Docker credentials"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Explicit build secret file; separate from runtime env_keys"), + ), + ); + operations::commands(command) +} + +pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result { + let targets = config.collect_all_targets_with_inheritance(); + let selected = if let Some(database) = database { + let mut matches = targets + .iter() + .filter(|target| target.fields.get("database").and_then(|v| v.as_str()) == Some(database)); + let target = matches + .next() + .context("database target is not in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "database target is ambiguous in local spacetime.json" + ); + target + } else { + let mut matches = targets.iter().filter(|target| target.container.is_some()); + let target = matches + .next() + .context("no container declaration in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "several container targets exist; select a DATABASE from local spacetime.json" + ); + target + }; + selected + .container + .clone() + .context("selected database has no container declaration; containers are not inherited") +} + +pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { + match args.subcommand().context("missing container command")? { + ("build", args) => exec_build(args).await, + ("exec", args) => execute::exec(&mut config, args).await, + ("url", args) => url::exec(&config, args).await, + ("logs", args) => logs::exec(&mut config, args).await, + (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, + _ => anyhow::bail!("unsupported container command"), + } +} + +pub async fn exec_build(args: &ArgMatches) -> Result<()> { + let project = args.get_one::("project_path").unwrap().canonicalize()?; + let loaded = find_and_load_with_env_from(args.get_one::("env").map(String::as_str), project)? + .context("spacetime.json not found")?; + let declaration = select(&loaded.config, args.get_one::("database").map(String::as_str))?; + let output = std::env::current_dir()?.join(args.get_one::("out_dir").unwrap()); + ensure!(!output.exists(), "output already exists: {}", output.display()); + let parent = output.parent().unwrap_or(Path::new(".")); + ensure!(parent.is_dir(), "output parent directory does not exist"); + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for secret in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = secret + .split_once('=') + .context("build secret must be NAME=FILE, not a value")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + let (os, architecture) = args.get_one::("platform").unwrap().split_once('/').unwrap(); + let cancel = CancellationToken::new(); + let prepared = { + let prepare = prepare_container( + &declaration, + &loaded.config_dir, + spacetimedb_lib::container::ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools, + parent, + &LocalRunner, + cancel.clone(), + ); + tokio::pin!(prepare); + tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; + cancel.cancel(); + let _ = prepare.await; + anyhow::bail!("container build cancelled; no output was published"); + } + } + }; + let digest = prepared.metadata.manifest.digest; + prepared.persist(&output)?; + println!("Prepared {digest} at {}", output.display()); + Ok(()) +} diff --git a/crates/cli/src/subcommands/container/exec.rs b/crates/cli/src/subcommands/container/exec.rs new file mode 100644 index 00000000000..71f8768812b --- /dev/null +++ b/crates/cli/src/subcommands/container/exec.rs @@ -0,0 +1,130 @@ +//! One invocation owns one exec socket. A lost connection is never replayed. +#[path = "exec/session.rs"] +mod session; +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[path = "exec/terminal.rs"] +mod terminal; +#[cfg(windows)] +#[path = "exec/terminal_windows.rs"] +mod terminal; +#[cfg(test)] +#[path = "exec/tests.rs"] +mod tests; + +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_lib::container::{ + exec::ExecStart, + operations::{ContainerStatus, DesiredState, ObservedState}, +}; +use std::collections::BTreeMap; + +pub(super) fn cli() -> Command { + Command::new("exec") + .about("Run a literal command in the current running container") + .after_help("Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux, macOS, and Windows terminals are supported. Windows requires an attached VT-capable console for --tty; inherited asynchronous seekable files are unsupported. A lost connection does not establish that the process exited.") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg(Arg::new("interactive").short('i').long("interactive").action(ArgAction::SetTrue).help("Forward stdin and send EOF when it closes")) + .arg(Arg::new("tty").short('t').long("tty").requires("interactive").action(ArgAction::SetTrue).help("Allocate a PTY using this foreground terminal's dimensions")) + .arg(Arg::new("workdir").long("workdir").help("Absolute working directory inside the container")) + .arg(Arg::new("environment").short('e').long("env").action(ArgAction::Append).value_name("NAME=VALUE").help("Override a process environment variable; platform keys are reserved")) + .arg(Arg::new("argv").required(true).num_args(1..).last(true).value_name("COMMAND").help("Executable and literal arguments; no shell expansion")) +} + +fn start(args: &ArgMatches, generation: u64) -> Result { + let mut environment = BTreeMap::new(); + for value in args.get_many::("environment").into_iter().flatten() { + let (key, value) = value + .split_once('=') + .context("exec environment overrides require NAME=VALUE")?; + ensure!( + environment.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate exec environment override" + ); + } + let start = ExecStart { + generation, + argv: args + .get_many::("argv") + .context("exec command is required")? + .cloned() + .collect(), + working_directory: args.get_one::("workdir").cloned(), + environment, + stdin: args.get_flag("interactive"), + terminal: None, + }; + start + .validate() + .context("invalid exec command, directory, or environment override")?; + Ok(start) +} + +fn running_generation(status: &ContainerStatus) -> Result { + ensure!(status.published, "database has no published container"); + let operation = status.operational.as_ref().context("container is not running")?; + let instance = operation + .current_instance + .as_ref() + .context("container is not running")?; + ensure!( + operation.desired_state == DesiredState::Running + && operation.generation != 0 + && instance.generation == operation.generation + && matches!(instance.state, ObservedState::Running | ObservedState::Ready), + "container is not running at the current generation" + ); + Ok(operation.generation) +} + +pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Result<()> { + // Reject unsupported terminal implementations before login, status, or a socket. + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = (config, args); + anyhow::bail!("container exec terminal support is not implemented on this host platform"); + } + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + { + let mut start = start(args, 1)?; + if args.get_flag("tty") { + start.terminal = Some(terminal::dimensions()?); + } + // Classify inherited Windows handles before any login or network I/O. + #[cfg(windows)] + let prepared = terminal::Prepared::stdio(start.stdin, start.terminal.is_some())?; + let selection = args.get_one::("server").map(String::as_str); + let origin = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let mut auth = auth.to_header().context("container exec requires a login")?; + auth.set_sensitive(true); + let client = super::operations::ContainerClient::new(origin, auth.clone())?; + let status = client + .status(args.get_one::("database").context("database is required")?) + .await?; + start.generation = running_generation(&status)?; + let url = client.url(status.database_identity.to_hex().as_ref(), "exec")?; + #[cfg(not(windows))] + let signals = terminal::signals(start.terminal.is_some())?; + #[cfg(not(windows))] + let (mut terminal, mut io) = terminal::Terminal::stdio(start.stdin, start.terminal.is_some())?; + #[cfg(windows)] + let (mut terminal, mut io) = prepared.start()?; + #[cfg(windows)] + let signals = terminal.signals()?; + let result = session::run(url, auth, status.database_identity, start, &mut io, signals).await; + // Always join and restore the terminal, including a failed handshake. + let cleanup = terminal.finish(); + let code = match (result, cleanup) { + (Ok(code), Ok(())) => code, + (Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error), + (Err(error), Err(cleanup)) => return Err(error.context(format!("terminal cleanup also failed: {cleanup}"))), + }; + if code != 0 { + return Err(crate::ExitWithCode(std::process::ExitCode::from(code)).into()); + } + Ok(()) + } +} diff --git a/crates/cli/src/subcommands/container/exec/session.rs b/crates/cli/src/subcommands/container/exec/session.rs new file mode 100644 index 00000000000..dc590e15a37 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/session.rs @@ -0,0 +1,227 @@ +use anyhow::{bail, ensure, Context, Result}; +use futures::{SinkExt, Stream, StreamExt}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION, SEC_WEBSOCKET_PROTOCOL}, + Url, +}; +use spacetimedb_lib::{container::exec::*, Identity}; +use std::time::Duration; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::{ + connect_async_with_config, + tungstenite::{client::IntoClientRequest, protocol::WebSocketConfig, Message}, +}; + +const NETWORK_WAIT: Duration = Duration::from_secs(10); +pub(super) enum Input { + Data(Vec), + Eof, +} +pub(super) struct Output { + pub stream: OutputStream, + pub bytes: Vec, + pub completed: oneshot::Sender>, +} +pub(super) type OutputStream = spacetimedb_lib::container::exec::Stream; +pub(super) struct OutputSender { + pub sender: mpsc::Sender, + pub wake: Option>, +} +impl OutputSender { + async fn send(&self, value: Output) -> Result<()> { + self.sender.send(value).await.context("terminal output stopped")?; + if let Some(wake) = &self.wake { + wake(); + } + Ok(()) + } +} +pub(super) struct Io { + pub input: mpsc::Receiver, + pub output: OutputSender, + pub completion: oneshot::Receiver>, +} + +pub(super) async fn run( + url: Url, + authorization: HeaderValue, + identity: Identity, + start: ExecStart, + io: &mut Io, + signals: S, +) -> Result +where + S: Stream> + Unpin, +{ + // Terminal failure/cancellation also owns the connection and Ready phases. + // The inline socket future drops before the caller joins native I/O workers. + let Io { + input, + output, + completion, + } = io; + tokio::select! { + result = run_socket(url, authorization, identity, start, input, output, signals) => result, + result = completion => { + result.context("terminal worker stopped")??; + bail!("terminal worker stopped before observed exec exit") + } + } +} + +async fn run_socket( + mut url: Url, + authorization: HeaderValue, + identity: Identity, + start: ExecStart, + input: &mut mpsc::Receiver, + output: &OutputSender, + mut signals: S, +) -> Result +where + S: Stream> + Unpin, +{ + start.validate()?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "invalid exec endpoint" + ); + let scheme = match url.scheme() { + "http" => "ws", + "https" => "wss", + _ => bail!("invalid exec endpoint"), + }; + url.set_scheme(scheme) + .map_err(|_| anyhow::anyhow!("invalid exec endpoint"))?; + url.query_pairs_mut() + .append_pair("generation", &start.generation.to_string()); + let mut request = url + .as_str() + .into_client_request() + .map_err(|_| anyhow::anyhow!("invalid exec endpoint"))?; + request.headers_mut().insert(AUTHORIZATION, authorization); + request + .headers_mut() + .insert(SEC_WEBSOCKET_PROTOCOL, HeaderValue::from_static(SUBPROTOCOL)); + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_CONTROL_BYTES)) + .max_frame_size(Some(MAX_CONTROL_BYTES)) + .write_buffer_size(0) + .max_write_buffer_size(MAX_CONTROL_BYTES + MAX_BINARY_BYTES + 1024); + let (mut socket, response) = + tokio::time::timeout(NETWORK_WAIT, connect_async_with_config(request, Some(config), true)) + .await + .map_err(|_| anyhow::anyhow!("exec connection timed out; the command was not replayed"))? + .map_err(|_| anyhow::anyhow!("exec connection was rejected or failed"))?; + ensure!( + response.headers().get_all(SEC_WEBSOCKET_PROTOCOL).iter().count() == 1 + && response + .headers() + .get(SEC_WEBSOCKET_PROTOCOL) + .is_some_and(|value| value == SUBPROTOCOL), + "server did not negotiate the exec protocol" + ); + send(&mut socket, text(&ClientControl::Start(start.clone()))?).await?; + tokio::time::timeout(NETWORK_WAIT, async { + loop { + match socket + .next() + .await + .context("exec closed before Ready; process outcome is unknown")? + .map_err(|_| anyhow::anyhow!("exec transport failed before Ready; process outcome is unknown"))? + { + Message::Text(value) => match ServerControl::decode(value.as_bytes())? { + ServerControl::Ready(ready) => { + ensure!( + ready.database_identity == identity + && ready.generation == start.generation + && ready.tty == start.terminal.is_some(), + "exec Ready did not match the selected instance" + ); + return Ok(()); + } + ServerControl::Error { .. } => bail!("container exec was rejected"), + _ => bail!("invalid exec response before Ready"), + }, + Message::Ping(bytes) => send(&mut socket, Message::Pong(bytes)).await?, + Message::Pong(_) => {} + _ => bail!("invalid exec response before Ready"), + } + } + }) + .await + .map_err(|_| anyhow::anyhow!("exec Ready timed out; process outcome is unknown"))??; + + let (mut sink, mut source) = socket.split(); + let (pong_tx, mut pong_rx) = mpsc::channel(1); + let reader = async { + while let Some(message) = source.next().await { + match message.map_err(|_| anyhow::anyhow!("exec transport failed; process outcome is unknown"))? { + Message::Binary(frame) => { + let (stream, bytes) = decode_output(&frame)?; + ensure!( + start.terminal.is_none() || stream == OutputStream::Stdout, + "unexpected PTY stderr channel" + ); + let (completed, written) = oneshot::channel(); + output + .send(Output { + stream, + bytes: bytes.to_vec(), + completed, + }) + .await + .context("terminal output stopped")?; + written.await.context("terminal output stopped")??; + } + Message::Text(value) => match ServerControl::decode(value.as_bytes())? { + ServerControl::Exit { exit_code } => { + return u8::try_from(exit_code).context("invalid observed exec exit code") + } + ServerControl::Error { .. } => bail!("container exec failed; process outcome is unknown"), + ServerControl::Ready(_) => bail!("duplicate exec Ready"), + }, + Message::Ping(bytes) => pong_tx.send(bytes).await.context("exec writer stopped")?, + Message::Pong(_) => {} + _ => bail!("exec closed without observed exit; process outcome is unknown"), + } + } + bail!("exec closed without observed exit; process outcome is unknown") + }; + let writer = async { + let mut stdin_open = start.stdin; + loop { + let message = tokio::select! { + value = input.recv(), if stdin_open => match value.context("terminal input stopped")? { + Input::Data(bytes) => Message::Binary(encode_data(OutputStream::Stdin, &bytes)?.into()), + Input::Eof => { stdin_open = false; text(&ClientControl::StdinEof)? }, + }, + value = signals.next() => text(&value.context("terminal signals stopped")??)?, + Some(bytes) = pong_rx.recv() => Message::Pong(bytes), + }; + send(&mut sink, message).await?; + } + #[allow(unreachable_code)] + Ok::(0) + }; + // These are owned, inline futures, not spawned tasks. The losing future and + // both socket halves are dropped before the caller joins its terminal worker. + tokio::select! { + result = reader => result, + result = writer => result, + } +} + +fn text(value: &ClientControl) -> Result { + value.validate()?; + Ok(Message::Text(serde_json::to_string(value)?.into())) +} +async fn send(sink: &mut S, message: Message) -> Result<()> +where + S: futures::Sink + Unpin, +{ + tokio::time::timeout(NETWORK_WAIT, sink.send(message)) + .await + .map_err(|_| anyhow::anyhow!("exec write timed out; process outcome is unknown"))? + .map_err(|_| anyhow::anyhow!("exec write failed; process outcome is unknown")) +} diff --git a/crates/cli/src/subcommands/container/exec/terminal.rs b/crates/cli/src/subcommands/container/exec/terminal.rs new file mode 100644 index 00000000000..b0fa92e61ec --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal.rs @@ -0,0 +1,404 @@ +//! Standard descriptors share open-file descriptions with their duplicates. +//! Save all flags before changing any, and restore them only after the one I/O +//! worker has joined. Nonblocking writes are necessary even after select: another +//! writer can fill a pipe between readiness and write. Never use Tokio stdin's +//! uncancellable blocking worker here. +use super::session::{Input, Io, Output, OutputSender, OutputStream}; +use anyhow::{bail, ensure, Context, Result}; +use futures::{stream, Stream, StreamExt}; +use rustix::{ + event::{fd_set_insert, fd_set_num_elements, select, FdSetElement, Timespec}, + fd::{AsRawFd, OwnedFd}, + fs::{fcntl_getfl, fcntl_setfl, OFlags}, + io::{read, write, Errno}, + termios::{tcgetattr, tcgetpgrp, tcgetwinsize, tcsetattr, OptionalActions, Termios}, +}; +use spacetimedb_lib::container::exec::{ClientControl, TerminalSize, MAX_DATA_BYTES}; +use std::{ + os::unix::net::UnixStream, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::JoinHandle, +}; +use tokio::sync::{mpsc, oneshot}; + +pub(super) fn dimensions() -> Result { + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); + ensure!( + tcgetpgrp(&stdin).ok() == Some(rustix::process::getpgrp()) + && tcgetpgrp(&stdout).ok() == Some(rustix::process::getpgrp()), + "PTY requires a foreground-owned stdin and stdout terminal" + ); + let size = tcgetwinsize(&stdout).context("cannot read terminal dimensions")?; + let size = TerminalSize { + rows: size.ws_row, + columns: size.ws_col, + }; + size.validate().context("invalid terminal dimensions")?; + Ok(size) +} + +type Signals = Pin> + Send>>; +pub(super) fn signals(tty: bool) -> Result { + use tokio::signal::unix::{signal, SignalKind}; + let mappings = [ + (SignalKind::hangup(), 1), + (SignalKind::interrupt(), 2), + (SignalKind::quit(), 3), + (SignalKind::terminate(), 15), + (SignalKind::user_defined1(), 10), + (SignalKind::user_defined2(), 12), + ]; + let mut streams: Vec = Vec::new(); + for (kind, linux) in mappings { + let signal = signal(kind).context("cannot install exec signal handler")?; + streams.push(Box::pin(stream::unfold(signal, move |mut signal| async move { + signal.recv().await.map(|()| (Ok(ClientControl::Signal(linux)), signal)) + }))); + } + if tty { + let signal = signal(SignalKind::window_change()).context("cannot install terminal resize handler")?; + streams.push(Box::pin(stream::unfold(signal, |mut signal| async move { + signal + .recv() + .await + .map(|()| (dimensions().map(ClientControl::Resize), signal)) + }))); + } + Ok(stream::select_all(streams).boxed()) +} + +struct Descriptors { + files: [OwnedFd; 3], + flags: [OFlags; 3], + terminal: Option, + restored: bool, +} +impl Descriptors { + fn prepare(files: [OwnedFd; 3], tty: bool) -> Result { + let flags = [ + fcntl_getfl(&files[0])?, + fcntl_getfl(&files[1])?, + fcntl_getfl(&files[2])?, + ]; + let mut this = Self { + files, + flags, + terminal: None, + restored: false, + }; + if tty { + ensure!( + tcgetpgrp(&this.files[0]).ok() == Some(rustix::process::getpgrp()) + && tcgetpgrp(&this.files[1]).ok() == Some(rustix::process::getpgrp()), + "PTY requires a foreground-owned terminal" + ); + let original = tcgetattr(&this.files[0])?; + let mut raw = original.clone(); + raw.make_raw(); + this.terminal = Some(original); + tcsetattr(&this.files[0], OptionalActions::Now, &raw)?; + } + for (fd, flags) in this.files.iter().zip(this.flags) { + fcntl_setfl(fd, flags | OFlags::NONBLOCK).context("cannot enable cancellable terminal I/O")?; + } + Ok(this) + } + fn restore(&mut self) -> Result<()> { + if self.restored { + return Ok(()); + } + let mut failed = false; + if let Some(terminal) = &self.terminal { + failed |= tcsetattr(&self.files[0], OptionalActions::Now, terminal).is_err(); + } + for (fd, flags) in self.files.iter().zip(self.flags) { + failed |= fcntl_setfl(fd, flags).is_err(); + } + self.restored = !failed; + ensure!(!failed, "could not restore terminal settings or descriptor flags"); + Ok(()) + } +} +impl Drop for Descriptors { + fn drop(&mut self) { + let _ = self.restore(); + } +} + +pub(super) struct Terminal { + stop: Arc, + wake: Arc, + worker: Option>>, +} +impl Terminal { + pub(super) fn stdio(stdin: bool, tty: bool) -> Result<(Self, Io)> { + Self::open( + [ + rustix::io::fcntl_dupfd_cloexec(std::io::stdin(), 3)?, + rustix::io::fcntl_dupfd_cloexec(std::io::stdout(), 3)?, + rustix::io::fcntl_dupfd_cloexec(std::io::stderr(), 3)?, + ], + stdin, + tty, + ) + } + fn open(files: [OwnedFd; 3], stdin: bool, tty: bool) -> Result<(Self, Io)> { + let mut descriptors = Descriptors::prepare(files, tty)?; + let (wake, wake_reader) = UnixStream::pair()?; + wake.set_nonblocking(true)?; + wake_reader.set_nonblocking(true)?; + let wake = Arc::new(wake); + let stop = Arc::new(AtomicBool::new(false)); + let (input_tx, input) = mpsc::channel(1); + let (output_tx, output) = mpsc::channel(1); + let (completed, completion) = oneshot::channel(); + let stopped = stop.clone(); + let worker = std::thread::Builder::new() + .name("container-exec-terminal".into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + pump(&descriptors.files, &wake_reader, stopped, stdin, input_tx, output) + })) + .unwrap_or_else(|_| Err(anyhow::anyhow!("terminal I/O worker panicked"))); + let restored = descriptors.restore(); + let result = match (result, restored) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(error), Err(restored)) => { + Err(error.context(format!("terminal restoration also failed: {restored}"))) + } + }; + let _ = completed.send( + result + .as_ref() + .map(|_| ()) + .map_err(|_| anyhow::anyhow!("terminal I/O failed")), + ); + result + }) + .context("cannot start terminal worker")?; + let notification = wake.clone(); + let output = OutputSender { + sender: output_tx, + wake: Some(Arc::new(move || { + let _ = write(&*notification, &[1]); + })), + }; + Ok(( + Self { + stop, + wake, + worker: Some(worker), + }, + Io { + input, + output, + completion, + }, + )) + } + pub(super) fn finish(&mut self) -> Result<()> { + self.stop.store(true, Ordering::Release); + let _ = write(&*self.wake, &[1]); + if let Some(worker) = self.worker.take() { + worker + .join() + .map_err(|_| anyhow::anyhow!("terminal worker join failed"))??; + } + Ok(()) + } +} +impl Drop for Terminal { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +fn pump( + files: &[OwnedFd; 3], + wake: &UnixStream, + stop: Arc, + mut stdin: bool, + input: mpsc::Sender, + mut output: mpsc::Receiver, +) -> Result<()> { + let mut incoming = None; + let mut outgoing: Option<(Output, usize)> = None; + let mut bytes = vec![0; MAX_DATA_BYTES]; + while !stop.load(Ordering::Acquire) { + if outgoing.is_none() { + outgoing = output.try_recv().ok().map(|value| (value, 0)); + } + let mut progress = false; + if let Some((value, offset)) = &mut outgoing { + let fd = match value.stream { + OutputStream::Stdout => &files[1], + OutputStream::Stderr => &files[2], + _ => bail!("invalid terminal output channel"), + }; + match write(fd, &value.bytes[*offset..]) { + Ok(0) => bail!("terminal output closed"), + Ok(count) => { + *offset += count; + progress = true; + } + Err(Errno::AGAIN | Errno::INTR) => {} + Err(_) => bail!("terminal output failed"), + } + if *offset == value.bytes.len() { + let (value, _) = outgoing.take().unwrap(); + let _ = value.completed.send(Ok(())); + } + } + if stdin && incoming.is_none() { + match read(&files[0], &mut bytes) { + Ok(0) => { + incoming = Some(Input::Eof); + stdin = false; + } + Ok(count) => { + incoming = Some(Input::Data(bytes[..count].to_vec())); + progress = true; + } + Err(Errno::AGAIN | Errno::INTR) => {} + Err(_) => bail!("terminal input failed"), + } + } + if let Some(value) = incoming.take() { + match input.try_send(value) { + Ok(()) => progress = true, + Err(mpsc::error::TrySendError::Full(value)) => incoming = Some(value), + Err(mpsc::error::TrySendError::Closed(_)) => bail!("terminal input consumer stopped"), + } + } + if progress { + continue; + } + let write_fd = outgoing + .as_ref() + .map(|(value, _)| files[if value.stream == OutputStream::Stdout { 1 } else { 2 }].as_raw_fd()); + let read_fd = (stdin && incoming.is_none()).then(|| files[0].as_raw_fd()); + let max = [Some(wake.as_raw_fd()), read_fd, write_fd] + .into_iter() + .flatten() + .max() + .unwrap() + + 1; + let elements = fd_set_num_elements(max as usize, 3); + let mut reads = vec![FdSetElement::default(); elements]; + let mut writes = reads.clone(); + fd_set_insert(&mut reads, wake.as_raw_fd()); + if let Some(fd) = read_fd { + fd_set_insert(&mut reads, fd); + } + if let Some(fd) = write_fd { + fd_set_insert(&mut writes, fd); + } + // select supports macOS terminals, unlike poll. Every registered FD is + // owned for the worker's entire lifetime; the sets use its exact bound. + let result = unsafe { + select( + max, + Some(&mut reads), + Some(&mut writes), + None, + Some(&Timespec { + tv_sec: 0, + tv_nsec: 20_000_000, + }), + ) + }; + if let Err(error) = result + && error != Errno::INTR + { + bail!("terminal readiness failed"); + } + let mut notifications = [0; 64]; + while read(wake, &mut notifications).is_ok_and(|count| count != 0) {} + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + #[tokio::test] + async fn cancelled_backpressured_terminal_joins_and_restores_shared_descriptor_flags() { + let (input, _input_peer) = UnixStream::pair().unwrap(); + let (output, _output_peer) = UnixStream::pair().unwrap(); + output.set_nonblocking(true).unwrap(); + let block = [0; 8192]; + while write(&output, &block).is_ok() {} + output.set_nonblocking(false).unwrap(); + let input_flags = fcntl_getfl(&input).unwrap(); + let output_flags = fcntl_getfl(&output).unwrap(); + let (mut terminal, io) = Terminal::open( + [ + rustix::io::dup(&input).unwrap(), + rustix::io::dup(&output).unwrap(), + rustix::io::dup(&output).unwrap(), + ], + true, + false, + ) + .unwrap(); + assert!(fcntl_getfl(&output).unwrap().contains(OFlags::NONBLOCK)); + let (completed, mut written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![42; MAX_DATA_BYTES], + completed, + }) + .await + .unwrap(); + if let Some(wake) = &io.output.wake { + wake(); + } + assert!(tokio::time::timeout(Duration::from_millis(30), &mut written) + .await + .is_err()); + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(terminal.worker.is_none()); + assert_eq!(fcntl_getfl(&input).unwrap(), input_flags); + assert_eq!(fcntl_getfl(&output).unwrap(), output_flags); + io.completion.await.unwrap().unwrap(); + assert!(written.await.is_err()); + terminal.finish().unwrap(); + } + + #[tokio::test] + async fn terminal_drop_joins_idle_input_and_refuses_unowned_pty_without_flag_changes() { + let (input, _input_peer) = UnixStream::pair().unwrap(); + let (output, _output_peer) = UnixStream::pair().unwrap(); + let flags = fcntl_getfl(&input).unwrap(); + let files = || { + [ + rustix::io::dup(&input).unwrap(), + rustix::io::dup(&output).unwrap(), + rustix::io::dup(&output).unwrap(), + ] + }; + assert!(Terminal::open(files(), true, true).is_err()); + assert_eq!(fcntl_getfl(&input).unwrap(), flags); + let (terminal, io) = Terminal::open(files(), true, false).unwrap(); + drop(terminal); + io.completion.await.unwrap().unwrap(); + assert_eq!(fcntl_getfl(&input).unwrap(), flags); + assert!(!fcntl_getfl(&output).unwrap().contains(OFlags::NONBLOCK)); + } +} + +#[cfg(test)] +#[path = "terminal_pty_tests.rs"] +mod pty_tests; diff --git a/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs b/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs new file mode 100644 index 00000000000..c9aa52e60b4 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs @@ -0,0 +1,163 @@ +//! A child session owns its own controlling PTY; the test runner's terminal and +//! process group are never modified. The parent retains and waits every child. +use super::*; +use rustix::{ + fs::{open, Mode}, + pty::{grantpt, openpt, ptsname, unlockpt, OpenptFlags}, + termios::{tcsetwinsize, LocalModes, Winsize}, +}; +use std::{ + fs::File, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +struct ChildOwner(Child); +impl Drop for ChildOwner { + fn drop(&mut self) { + if self.0.try_wait().ok().flatten().is_none() { + let _ = self.0.kill(); + } + let _ = self.0.wait(); + } +} + +#[test] +fn foreground_pty_restores_after_normal_error_and_cancel() { + for mode in ["normal", "error", "cancel"] { + let master = openpt(OpenptFlags::RDWR | OpenptFlags::NOCTTY).unwrap(); + rustix::io::fcntl_setfd(&master, rustix::io::FdFlags::CLOEXEC).unwrap(); + grantpt(&master).unwrap(); + unlockpt(&master).unwrap(); + let path = ptsname(&master, Vec::new()).unwrap(); + let slave = open(&path, OFlags::RDWR | OFlags::NOCTTY | OFlags::CLOEXEC, Mode::empty()).unwrap(); + tcsetwinsize( + &slave, + Winsize { + ws_row: 48, + ws_col: 120, + ws_xpixel: 0, + ws_ypixel: 0, + }, + ) + .unwrap(); + // Prime Darwin's kernel FWASWRITTEN state before the snapshot; + // the child test harness writes to this same open-file description. + assert_eq!(write(&slave, b"owned PTY fixture\n").unwrap(), 18); + let before = configuration(tcgetattr(&slave).unwrap()); + let flags = fcntl_getfl(&slave).unwrap(); + let root = tempfile::tempdir().unwrap(); + let file: File = rustix::io::dup(&slave).unwrap().into(); + let child = Command::new(std::env::current_exe().unwrap()) + .args([ + "subcommands::container::execute::terminal::pty_tests::foreground_pty_child", + "--exact", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env_clear() + .env("SPACETIMEDB_EXEC_PTY_CHILD", mode) + .env("TMPDIR", root.path()) + .current_dir(root.path()) + .stdin(Stdio::from(file.try_clone().unwrap())) + .stdout(Stdio::from(file.try_clone().unwrap())) + .stderr(Stdio::from(file)) + .spawn() + .unwrap(); + let mut child = ChildOwner(child); + fcntl_setfl(&master, fcntl_getfl(&master).unwrap() | OFlags::NONBLOCK).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let mut captured = Vec::new(); + let mut verified = false; + let mut buffer = [0; 4096]; + let status = loop { + while let Ok(count) = read(&master, &mut buffer) { + if count == 0 { + break; + } + captured.extend_from_slice(&buffer[..count]); + assert!(captured.len() < 64 * 1024); + } + if !verified + && captured + .windows(b"PTY_RESTORED".len()) + .any(|text| text == b"PTY_RESTORED") + { + assert_eq!(configuration(tcgetattr(&slave).unwrap()), before, "mode {mode}"); + assert_eq!(fcntl_getfl(&slave).unwrap(), flags, "mode {mode}"); + assert_eq!(write(&master, b"y\n").unwrap(), 2); + verified = true; + } + if let Some(status) = child.0.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "owned PTY child timed out"); + std::thread::sleep(Duration::from_millis(5)); + }; + // Keep the session leader alive for the independent slave inspection: + // macOS revokes that slave when the controlling session exits. + // Then positively wait, even after try_wait. + child.0.wait().unwrap(); + assert!(status.success(), "mode {mode}: {}", String::from_utf8_lossy(&captured)); + assert!(verified, "mode {mode} did not acknowledge restored state"); + assert_eq!(fcntl_getfl(&slave).unwrap(), flags, "mode {mode}"); + } +} + +#[tokio::test] +#[ignore = "owned child of foreground_pty_restores_after_normal_error_and_cancel"] +async fn foreground_pty_child() { + let mode = std::env::var("SPACETIMEDB_EXEC_PTY_CHILD").expect("requires owned PTY parent"); + assert!(["normal", "error", "cancel"].contains(&mode.as_str())); + rustix::process::setsid().unwrap(); + rustix::process::ioctl_tiocsctty(std::io::stdin()).unwrap(); + rustix::termios::tcsetpgrp(std::io::stdin(), rustix::process::getpgrp()).unwrap(); + assert_eq!(dimensions().unwrap(), TerminalSize { rows: 48, columns: 120 }); + let before = configuration(tcgetattr(std::io::stdin()).unwrap()); + let flags = fcntl_getfl(std::io::stdin()).unwrap(); + let (mut terminal, io) = Terminal::stdio(true, true).unwrap(); + assert!(!tcgetattr(std::io::stdin()) + .unwrap() + .local_modes + .intersects(LocalModes::ICANON | LocalModes::ECHO)); + match mode.as_str() { + "normal" => terminal.finish().unwrap(), + "cancel" => drop(terminal), + "error" => { + let (completed, _written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdin, + bytes: vec![1], + completed, + }) + .await + .unwrap(); + if let Some(wake) = &io.output.wake { + wake(); + } + assert!(io.completion.await.unwrap().is_err()); + assert!(terminal.finish().is_err()); + } + _ => unreachable!(), + } + assert_eq!(configuration(tcgetattr(std::io::stdin()).unwrap()), before); + assert_eq!(fcntl_getfl(std::io::stdin()).unwrap(), flags); + use std::io::{Read, Write}; + println!("PTY_RESTORED"); + std::io::stdout().flush().unwrap(); + let mut ack = [0]; + std::io::stdin().read_exact(&mut ack).unwrap(); + assert_eq!(ack, [b'y']); +} + +fn configuration(mut termios: Termios) -> String { + // PENDIN is queued-input state, not a configured terminal mode. macOS + // sets it when ICANON is restored, even for an empty queue. Do not flush + // user input or change production restoration to erase this kernel state. + // https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/tty.c + termios.local_modes.remove(LocalModes::PENDIN); + format!("{termios:?}") +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows.rs b/crates/cli/src/subcommands/container/exec/terminal_windows.rs new file mode 100644 index 00000000000..dcff29de6fe --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows.rs @@ -0,0 +1,373 @@ +//! Windows stdio workers remain owned through cancellation and console restoration. +//! Inherited synchronous handles cannot be converted to overlapped handles by dup. +#[path = "terminal_windows/control.rs"] +mod control; +#[path = "terminal_windows/io.rs"] +mod io; +#[cfg(test)] +#[path = "terminal_windows/tests.rs"] +mod tests; + +use super::session::{Input, Io, Output, OutputSender, OutputStream}; +use anyhow::{bail, ensure, Context, Result}; +use futures::{stream, Stream, StreamExt}; +use io::{ConsoleState, Event, File, Kind}; +use spacetimedb_lib::container::exec::{ClientControl, TerminalSize, MAX_DATA_BYTES}; +use std::{ + os::windows::io::AsRawHandle, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Condvar, Mutex, MutexGuard, + }, + thread::JoinHandle, + time::Duration, +}; +use tokio::sync::{mpsc, oneshot}; +use windows_sys::Win32::{ + Foundation::*, + System::{Console::*, IO::CancelSynchronousIo}, +}; + +fn check(ok: BOOL) -> Result<()> { + ensure!(ok != 0, "Windows terminal operation failed"); + Ok(()) +} +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|error| error.into_inner()) +} + +struct Shared { + stop: AtomicBool, + wake: Event, + restored: Event, + failure: Mutex>>>, + controls: mpsc::Sender>, + callbacks: Mutex, + callbacks_done: Condvar, +} +impl Shared { + fn stopped(&self) -> bool { + self.stop.load(Ordering::Acquire) + } + fn stop(&self) { + self.stop.store(true, Ordering::Release); + self.wake.set(); + } + fn fail(&self) { + self.stop(); + if let Some(sender) = lock(&self.failure).take() { + let _ = sender.send(Err(anyhow::anyhow!("Windows terminal stopped"))); + } + } +} + +pub(super) fn dimensions() -> Result { + let mut input_mode = 0; + let mut info: CONSOLE_SCREEN_BUFFER_INFO = unsafe { std::mem::zeroed() }; + unsafe { + check(GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &mut input_mode)) + .context("PTY requires attached console input")?; + check(GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &mut info)) + .context("PTY requires attached console output")?; + } + let size = TerminalSize { + rows: u16::try_from(i32::from(info.srWindow.Bottom) - i32::from(info.srWindow.Top) + 1)?, + columns: u16::try_from(i32::from(info.srWindow.Right) - i32::from(info.srWindow.Left) + 1)?, + }; + size.validate().context("invalid console dimensions")?; + Ok(size) +} + +pub(super) struct Prepared { + files: [Option; 3], + tty: bool, +} +impl Prepared { + pub(super) fn stdio(stdin: bool, tty: bool) -> Result { + let handles = unsafe { + [ + GetStdHandle(STD_INPUT_HANDLE), + GetStdHandle(STD_OUTPUT_HANDLE), + GetStdHandle(STD_ERROR_HANDLE), + ] + }; + Self::new( + [ + stdin.then(|| File::duplicate(handles[0])).transpose()?, + Some(File::duplicate(handles[1])?), + (!tty).then(|| File::duplicate(handles[2])).transpose()?, + ], + tty, + ) + } + fn new(files: [Option; 3], tty: bool) -> Result { + ensure!(files[1].is_some(), "terminal output is required"); + ensure!( + !tty || files[..2] + .iter() + .all(|file| file.as_ref().is_some_and(|file| file.kind == Kind::Console)), + "PTY requires attached console input and output" + ); + Ok(Self { files, tty }) + } + pub(super) fn start(self) -> Result<(Terminal, Io)> { + let (input_sender, input) = mpsc::channel(1); + let (output_sender, output) = mpsc::channel(1); + let (completed, completion) = oneshot::channel(); + let (control_sender, controls) = mpsc::channel(8); + let shared = Arc::new(Shared { + stop: AtomicBool::new(false), + wake: Event::new()?, + restored: Event::new()?, + failure: Mutex::new(Some(completed)), + controls: control_sender, + callbacks: Mutex::new(0), + callbacks_done: Condvar::new(), + }); + // Reserve callback/console ownership before any shared console mutation. + let registration = control::Registration::new(shared.clone())?; + let mut terminal = Terminal { + shared: shared.clone(), + workers: Vec::new(), + console: None, + registration: Some(registration), + controls: Some(controls), + tty: self.tty, + finished: false, + failed: false, + }; + terminal.console = Some(ConsoleState::prepare(&self.files, self.tty)?); + let [stdin, stdout, stderr] = self.files; + if let Some(stdin) = stdin { + terminal.spawn("container-exec-input", move |shared| { + read_input(stdin, input_sender, shared) + })?; + } + terminal.spawn("container-exec-output", move |shared| { + write_output(stdout.unwrap(), stderr, output, shared) + })?; + let notification = shared.clone(); + Ok(( + terminal, + Io { + input, + output: OutputSender { + sender: output_sender, + wake: Some(Arc::new(move || notification.wake.set())), + }, + completion, + }, + )) + } +} + +pub(super) struct Terminal { + shared: Arc, + workers: Vec>>, + console: Option, + registration: Option, + controls: Option>>, + tty: bool, + finished: bool, + failed: bool, +} +type Signals = Pin> + Send>>; +impl Terminal { + fn spawn(&mut self, name: &str, work: impl FnOnce(&Shared) -> Result<()> + Send + 'static) -> Result<()> { + let shared = self.shared.clone(); + self.workers.push( + std::thread::Builder::new() + .name(name.into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| work(&shared))) + .unwrap_or_else(|_| Err(anyhow::anyhow!("Windows terminal worker panicked"))); + if result.is_err() { + shared.fail(); + } + result + }) + .context("cannot start Windows terminal worker")?, + ); + Ok(()) + } + pub(super) fn signals(&mut self) -> Result { + let receiver = self.controls.take().context("terminal controls already consumed")?; + let mut streams: Vec = vec![Box::pin(stream::unfold(receiver, |mut receiver| async move { + receiver.recv().await.map(|control| (control, receiver)) + }))]; + if self.tty { + let initial = dimensions()?; + let interval = tokio::time::interval(Duration::from_millis(200)); + streams.push(Box::pin(stream::unfold( + (interval, initial), + |(mut interval, mut previous)| async move { + loop { + interval.tick().await; + match dimensions() { + Ok(size) if size == previous => {} + Ok(size) => { + previous = size; + return Some((Ok(ClientControl::Resize(size)), (interval, previous))); + } + Err(error) => return Some((Err(error), (interval, previous))), + } + } + }, + ))); + } + Ok(stream::select_all(streams).boxed()) + } + pub(super) fn finish(&mut self) -> Result<()> { + if self.finished { + ensure!(!self.failed, "Windows terminal cleanup previously failed"); + return Ok(()); + } + let mut errors = Vec::new(); + if let Some(registration) = &mut self.registration + && let Err(error) = registration.seal() + { + errors.push(error); + } + self.shared.stop(); + // Repeat cancellation until actual completion, including registration races. + while self.workers.iter().any(|worker| !worker.is_finished()) { + for worker in self.workers.iter().filter(|worker| !worker.is_finished()) { + unsafe { + CancelSynchronousIo(worker.as_raw_handle()); + } + } + std::thread::sleep(Duration::from_millis(10)); + } + for worker in self.workers.drain(..) { + match worker.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => errors.push(error), + Err(_) => errors.push(anyhow::anyhow!("Windows terminal worker join failed")), + } + } + if let Some(console) = &mut self.console + && let Err(error) = console.restore() + { + errors.push(error); + } + // A close callback waits on this event. Signal before waiting for it. + self.shared.restored.set(); + let mut count = lock(&self.shared.callbacks); + while *count != 0 { + count = self + .shared + .callbacks_done + .wait(count) + .unwrap_or_else(|error| error.into_inner()); + } + drop(count); + self.registration.take(); + self.finished = true; + self.failed = !errors.is_empty(); + if !errors.is_empty() { + let error = errors.remove(0); + return Err(error.context(format!( + "Windows terminal cleanup failed ({} additional failures)", + errors.len() + ))); + } + Ok(()) + } +} +impl Drop for Terminal { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +fn send_input(sender: &mpsc::Sender, mut value: Input, shared: &Shared) -> Result<()> { + while !shared.stopped() { + match sender.try_send(value) { + Ok(()) => return Ok(()), + Err(mpsc::error::TrySendError::Closed(_)) => bail!("terminal input consumer stopped"), + Err(mpsc::error::TrySendError::Full(returned)) => value = returned, + } + // No blocking queue send can prevent cancellation. Input consumption has + // no native wake hook, so this one bounded pending chunk polls at 20ms. + std::thread::sleep(Duration::from_millis(20)); + } + Ok(()) +} +fn read_input(file: File, sender: mpsc::Sender, shared: &Shared) -> Result<()> { + let mut bytes = vec![0u8; MAX_DATA_BYTES]; + let mut units = vec![0u16; MAX_DATA_BYTES / 4]; + let mut high_surrogate = None; + while !shared.stopped() { + let size = if file.kind == Kind::Console { + let count = file.read_console(&mut units, shared)?; + if shared.stopped() { + return Ok(()); + } + if count == 0 { + 0 + } else { + let text = decode_console(&units[..count], &mut high_surrogate)?; + bytes[..text.len()].copy_from_slice(text.as_bytes()); + if text.is_empty() { + continue; + } + text.len() + } + } else { + file.read(&mut bytes, shared)? + }; + if shared.stopped() { + return Ok(()); + } + if size == 0 { + ensure!(high_surrogate.is_none(), "incomplete console Unicode input"); + return send_input(&sender, Input::Eof, shared); + } + send_input(&sender, Input::Data(bytes[..size].to_vec()), shared)?; + } + Ok(()) +} +fn decode_console(units: &[u16], high: &mut Option) -> Result { + let mut combined = Vec::with_capacity(units.len() + 1); + combined.extend(high.take()); + combined.extend_from_slice(units); + if combined.last().is_some_and(|unit| (0xd800..=0xdbff).contains(unit)) { + *high = combined.pop(); + } + String::from_utf16(&combined).context("invalid console Unicode input") +} +fn write_output( + stdout: File, + stderr: Option, + mut receiver: mpsc::Receiver, + shared: &Shared, +) -> Result<()> { + while !shared.stopped() { + shared.wake.reset(); + match receiver.try_recv() { + Ok(value) => { + let file = match value.stream { + OutputStream::Stdout => &stdout, + OutputStream::Stderr => stderr.as_ref().context("terminal stderr is unavailable")?, + _ => bail!("invalid terminal output channel"), + }; + let mut offset = 0; + while offset < value.bytes.len() && !shared.stopped() { + let count = file.write(&value.bytes[offset..], shared)?; + if shared.stopped() { + return Ok(()); + } + ensure!(count != 0, "terminal output closed"); + offset += count; + } + if !shared.stopped() { + let _ = value.completed.send(Ok(())); + } + } + Err(mpsc::error::TryRecvError::Empty) => shared.wake.wait(20), + Err(mpsc::error::TryRecvError::Disconnected) => return Ok(()), + } + } + Ok(()) +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs new file mode 100644 index 00000000000..d9c78edcd2d --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs @@ -0,0 +1,385 @@ +//! The parent owns a ConPTY, an exact test child and a separately drained output +//! pipe. No desktop, inherited account configuration, or user console is needed. +use super::*; +use std::{ + ffi::OsStr, + os::windows::ffi::OsStrExt, + path::{Path, PathBuf}, +}; + +fn wide(value: &OsStr) -> Vec { + value.encode_wide().chain([0]).collect() +} +struct Child { + process: Option, + console: HPCON, + input: Option, + startup_pipes: Option<(OwnedHandle, OwnedHandle)>, + reader: Option>>>, + waited: bool, +} +impl Child { + fn start(root: &Path, mode: &str) -> Result { + let (input_read, input) = pipe(); + let (output_read, output_write) = pipe(); + let mut this = Self { + process: None, + console: 0, + input: Some(input), + startup_pipes: Some((input_read, output_write)), + reader: None, + waited: false, + }; + ensure!( + unsafe { + CreatePseudoConsole( + COORD { X: 120, Y: 48 }, + this.startup_pipes.as_ref().unwrap().0.as_raw_handle(), + this.startup_pipes.as_ref().unwrap().1.as_raw_handle(), + 0, + &mut this.console, + ) + } == 0, + "cannot create owned pseudoconsole" + ); + // Drain concurrently before launching a process that can write console output. + this.reader = Some(std::thread::spawn(move || { + let mut all = Vec::new(); + let mut bytes = [0; 4096]; + loop { + let mut count = 0; + let ok = unsafe { + ReadFile( + output_read.as_raw_handle(), + bytes.as_mut_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + ) + }; + if ok == 0 { + ensure!( + unsafe { GetLastError() } == ERROR_BROKEN_PIPE, + "owned console drain failed" + ); + break; + } + if count == 0 { + break; + } + if all.len() < 1024 * 1024 { + all.extend_from_slice(&bytes[..count as usize]); + } + } + Ok(all) + })); + let mut length = 0; + unsafe { + InitializeProcThreadAttributeList(null_mut(), 1, 0, &mut length); + } + ensure!(length > 0 && length <= 4096, "invalid owned process attribute size"); + let mut storage = vec![0usize; length.div_ceil(size_of::())]; + let attributes = storage.as_mut_ptr().cast(); + unsafe { + check(InitializeProcThreadAttributeList(attributes, 1, 0, &mut length))?; + } + struct Attributes(windows_sys::Win32::System::Threading::LPPROC_THREAD_ATTRIBUTE_LIST); + impl Drop for Attributes { + fn drop(&mut self) { + unsafe { + DeleteProcThreadAttributeList(self.0); + } + } + } + let attributes = Attributes(attributes); + unsafe { + check(UpdateProcThreadAttribute( + attributes.0, + 0, + PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE as usize, + this.console as *const _, + size_of::(), + null_mut(), + null(), + ))?; + } + let executable = std::env::current_exe()?; + let module = module_path!().split_once("::").unwrap().1; + let selected = format!("{module}::windows_console_child"); + let mut command = wide(OsStr::new(&format!( + "\"{}\" --exact {selected} --ignored --nocapture --test-threads=1", + executable.display() + ))); + // Inherit no endpoint, auth, proxy or CLI settings. SystemRoot is only + // the Windows DLL/runtime base; all writable locations belong to root. + let system = std::env::var_os("SystemRoot").context("missing Windows system directory")?; + let mut environment = Vec::new(); + for (key, value) in [ + ("STDB_EXEC_CONSOLE_MODE", OsStr::new(mode)), + ("STDB_EXEC_CONSOLE_ROOT", root.as_os_str()), + ("SystemRoot", system.as_os_str()), + ("TEMP", root.as_os_str()), + ("TMP", root.as_os_str()), + ] { + environment.extend(format!("{key}=").encode_utf16()); + environment.extend(value.encode_wide()); + environment.push(0); + } + environment.push(0); + let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() }; + startup.StartupInfo.cb = size_of::() as u32; + // Without this flag, Windows can duplicate the parent's redirected + // handles even with bInheritHandles=false. Explicit null standard + // handles let the pseudoconsole supply its own console handles. + // https://github.com/microsoft/terminal/discussions/15814 + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.lpAttributeList = attributes.0; + let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() }; + unsafe { + check(CreateProcessW( + wide(executable.as_os_str()).as_ptr(), + command.as_mut_ptr(), + null(), + null(), + 0, + EXTENDED_STARTUPINFO_PRESENT | CREATE_UNICODE_ENVIRONMENT, + environment.as_ptr().cast(), + wide(root.as_os_str()).as_ptr(), + &startup.StartupInfo, + &mut process, + ))?; + } + this.process = Some(unsafe { OwnedHandle::from_raw_handle(process.hProcess) }); + drop(unsafe { OwnedHandle::from_raw_handle(process.hThread) }); + this.startup_pipes.take(); + Ok(this) + } + fn complete(&mut self) -> Result<()> { + let process = self.process.as_ref().context("owned child not started")?; + let completed = unsafe { WaitForSingleObject(process.as_raw_handle(), 10_000) } == WAIT_OBJECT_0; + ensure!(completed, "owned console child timed out"); + self.waited = true; + let mut code = 1; + unsafe { + check(GetExitCodeProcess(process.as_raw_handle(), &mut code))?; + } + ensure!(code == 0, "owned console child failed"); + Ok(()) + } + fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); + if let Some(process) = &self.process + && !self.waited + { + unsafe { + // An early fixture failure may already have exited before the + // ready marker. Observe that exact child before forcing it. + self.waited = WaitForSingleObject(process.as_raw_handle(), 0) == WAIT_OBJECT_0; + if !self.waited { + errors.push(anyhow::anyhow!("owned console child required forced cleanup")); + if let Err(error) = check(TerminateProcess(process.as_raw_handle(), 1)) { + errors.push(error.context("cannot terminate owned console child")); + } + self.waited = WaitForSingleObject(process.as_raw_handle(), INFINITE) == WAIT_OBJECT_0; + } + } + if !self.waited { + errors.push(anyhow::anyhow!("owned console child wait failed")); + } + } + self.input.take(); + // Also close parent's startup copies on every partial-start failure; + // retaining output_write would keep the drain waiting after ConPTY close. + self.startup_pipes.take(); + // Keep the drain alive while closing ConPTY; close can emit final output. + if self.console != 0 { + unsafe { + ClosePseudoConsole(self.console); + } + self.console = 0; + } + if let Some(reader) = self.reader.take() { + match reader.join() { + Ok(Ok(_)) => (), + Ok(Err(error)) => errors.push(error), + Err(_) => errors.push(anyhow::anyhow!("owned console drain panicked")), + } + } + ensure!( + errors.is_empty(), + "{}", + errors + .iter() + .map(|error| format!("{error:#}")) + .collect::>() + .join("; ") + ); + Ok(()) + } +} +impl Drop for Child { + fn drop(&mut self) { + if let Err(error) = self.close() { + eprintln!("owned console cleanup failed: {error:#}"); + } + } +} + +#[test] +fn native_windows_conpty_normal_error_cancel_restore_console() { + let _owner = lock(&TEST_OWNER); + for mode in ["normal", "error", "cancel"] { + let root = tempfile::tempdir().unwrap(); + let mut child = Child::start(root.path(), mode).unwrap(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let deadline = Instant::now() + Duration::from_secs(10); + while !root.path().join("ready").exists() { + assert!(Instant::now() < deadline, "owned console child did not become ready"); + std::thread::sleep(Duration::from_millis(10)); + } + write(child.input.as_ref().unwrap(), "🦀\u{1b}[A\u{3}".as_bytes()); + assert_eq!( + unsafe { ResizePseudoConsole(child.console, COORD { X: 100, Y: 40 }) }, + 0 + ); + child.complete()?; + assert_eq!( + std::fs::read(root.path().join("restored")).unwrap(), + b"all console settings restored" + ); + Ok(()) + })); + let cleanup = child.close(); + finish(result, [cleanup]).unwrap(); + } +} + +#[test] +#[ignore = "exact owned ConPTY child, invoked by native_windows_conpty_normal_error_cancel_restore_console"] +fn windows_console_child() { + let root = PathBuf::from(std::env::var_os("STDB_EXEC_CONSOLE_ROOT").expect("owned console child only")); + let mode = std::env::var("STDB_EXEC_CONSOLE_MODE").unwrap(); + assert!(matches!(mode.as_str(), "normal" | "error" | "cancel")); + let handles = unsafe { + [ + GetStdHandle(STD_INPUT_HANDLE), + GetStdHandle(STD_OUTPUT_HANDLE), + GetStdHandle(STD_ERROR_HANDLE), + ] + }; + let original: Vec<_> = handles + .iter() + .enumerate() + .map(|(index, handle)| { + let mut value = 0; + unsafe { + assert!( + GetConsoleMode(*handle, &mut value) != 0, + "owned console standard handle {index} is not a console: Win32 error {}", + GetLastError() + ); + } + value + }) + .collect(); + let code_page = unsafe { GetConsoleOutputCP() }; + let initial = dimensions().unwrap(); + assert_eq!((initial.rows, initial.columns), (48, 120)); + let (mut terminal, mut io) = Prepared::stdio(true, true).unwrap().start().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut controls = { + let _entered = runtime.enter(); + terminal.signals().unwrap() + }; + let mut raw = 0; + unsafe { + check(GetConsoleMode(handles[0], &mut raw)).unwrap(); + } + assert_eq!( + raw & (ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT), + 0 + ); + assert_ne!(raw & ENABLE_VIRTUAL_TERMINAL_INPUT, 0); + std::fs::write(root.join("ready"), b"ready").unwrap(); + let result = runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(5), async { + let expected = "🦀\u{1b}[A\u{3}".as_bytes(); + let mut received = Vec::new(); + while received.len() < expected.len() { + let Input::Data(bytes) = io.input.recv().await.context("missing console input")? else { + bail!("early console EOF") + }; + received.extend(bytes); + } + ensure!(received == expected, "console UTF-8/VT input differs"); + loop { + if let Some(Ok(ClientControl::Resize(size))) = controls.next().await + && (size.rows, size.columns) == (40, 100) + { + break; + } + } + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: if mode == "error" { + OutputStream::Stdin + } else { + OutputStream::Stdout + }, + bytes: "owned 🦀 output\r\n".as_bytes().to_vec(), + completed, + }) + .await?; + terminal.shared.wake.set(); + if mode == "error" { + ensure!(written.await.is_err(), "invalid output channel accepted"); + } else { + written.await??; + } + Ok::<(), anyhow::Error>(()) + }) + .await? + }); + let cleanup = if mode == "cancel" { + drop(terminal); + Ok(()) + } else { + terminal.finish() + }; + result.unwrap(); + assert_eq!(cleanup.is_err(), mode == "error"); + for (handle, previous) in handles.iter().zip(original) { + let mut value = 0; + unsafe { + check(GetConsoleMode(*handle, &mut value)).unwrap(); + } + assert_eq!(value, previous); + } + assert_eq!(unsafe { GetConsoleOutputCP() }, code_page); + std::fs::write(root.join("restored"), b"all console settings restored").unwrap(); +} + +#[test] +fn native_windows_console_cleanup_reports_drain_failure() { + for panic in [false, true] { + let mut child = Child { + process: None, + console: 0, + input: None, + startup_pipes: None, + reader: Some(std::thread::spawn(move || { + assert!(!panic, "owned drain panic"); + bail!("owned drain failure") + })), + waited: false, + }; + let error = child.close().unwrap_err(); + assert!(error.to_string().contains(if panic { "panicked" } else { "failure" })); + assert!(child.reader.is_none()); + } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs new file mode 100644 index 00000000000..b7a09d5d68d --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs @@ -0,0 +1,97 @@ +use super::{check, lock, ClientControl, Shared}; +use anyhow::{ensure, Result}; +use std::sync::{Arc, Mutex}; +use windows_sys::Win32::{ + Foundation::{BOOL, FALSE, TRUE}, + System::{Console::*, Threading::INFINITE}, +}; + +static ACTIVE: Mutex, bool)>> = Mutex::new(None); +pub(super) struct Registration { + shared: Arc, + sealed: bool, +} +impl Registration { + pub fn new(shared: Arc) -> Result { + let mut active = lock(&ACTIVE); + ensure!(active.is_none(), "another Windows terminal owner is active"); + unsafe { + check(SetConsoleCtrlHandler(Some(handler), TRUE))?; + } + *active = Some((shared.clone(), true)); + Ok(Self { shared, sealed: false }) + } + pub fn seal(&mut self) -> Result<()> { + if self.sealed { + return Ok(()); + } + let mut active = lock(&ACTIVE); + if let Some((shared, admitted)) = active.as_mut() + && Arc::ptr_eq(shared, &self.shared) + { + *admitted = false; + } + self.sealed = true; + unsafe { check(SetConsoleCtrlHandler(Some(handler), FALSE)) } + } +} +impl Drop for Registration { + fn drop(&mut self) { + let _ = self.seal(); + let mut active = lock(&ACTIVE); + if active + .as_ref() + .is_some_and(|(shared, _)| Arc::ptr_eq(shared, &self.shared)) + { + *active = None; + } + } +} + +struct Callback(Arc); +impl Drop for Callback { + fn drop(&mut self) { + let mut count = lock(&self.0.callbacks); + *count -= 1; + self.0.callbacks_done.notify_all(); + } +} +unsafe extern "system" fn handler(event: u32) -> BOOL { + let callback = { + let active = lock(&ACTIVE); + let Some((shared, true)) = active.as_ref() else { + return FALSE; + }; + *lock(&shared.callbacks) += 1; + Callback(shared.clone()) + }; + // The OS owns this callback thread. Never unwind across the ABI boundary. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dispatch(event, &callback.0))).unwrap_or_else(|_| { + callback.0.fail(); + TRUE + }) +} +fn dispatch(event: u32, shared: &Shared) -> BOOL { + match event { + CTRL_C_EVENT | CTRL_BREAK_EVENT => { + let signal = if event == CTRL_C_EVENT { 2 } else { 3 }; + if shared.controls.try_send(Ok(ClientControl::Signal(signal))).is_err() { + shared.fail(); + } + TRUE + } + CTRL_CLOSE_EVENT | CTRL_LOGOFF_EVENT | CTRL_SHUTDOWN_EVENT => { + shared.fail(); + // Returning allows OS termination. Only acknowledge after joined I/O + // and restoration; an OS forced timeout cannot count as completion. + shared.restored.wait(INFINITE); + TRUE + } + _ => FALSE, + } +} + +#[cfg(test)] +pub(super) fn invoke(event: u32) -> BOOL { + unsafe { handler(event) } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs new file mode 100644 index 00000000000..20e30e1c949 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs @@ -0,0 +1,344 @@ +//! Buffers and OVERLAPPED storage outlive the exact operation, including cancellation. +use super::{check, Shared}; +use anyhow::{bail, ensure, Result}; +use std::{ + os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}, + ptr::{null, null_mut}, + sync::Arc, +}; +use windows_sys::{ + Wdk::Storage::FileSystem::{ + FileModeInformation, NtQueryInformationFile, FILE_MODE_INFORMATION, FILE_SYNCHRONOUS_IO_ALERT, + FILE_SYNCHRONOUS_IO_NONALERT, + }, + Win32::{ + Foundation::*, + Storage::FileSystem::*, + System::{Console::*, Threading::*, IO::*}, + }, +}; + +pub(super) struct Event(OwnedHandle); +impl Event { + pub fn new() -> Result { + // SAFETY: no security attributes, name, or inherited handle; owned once. + let raw = unsafe { CreateEventW(null(), 1, 0, null()) }; + ensure!(!raw.is_null(), "cannot create terminal event"); + Ok(Self(unsafe { OwnedHandle::from_raw_handle(raw) })) + } + pub fn raw(&self) -> HANDLE { + self.0.as_raw_handle() + } + pub fn set(&self) { + unsafe { + SetEvent(self.raw()); + } + } + pub fn reset(&self) { + unsafe { + ResetEvent(self.raw()); + } + } + pub fn wait(&self, millis: u32) { + unsafe { + WaitForSingleObject(self.raw(), millis); + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Kind { + Console, + Synchronous, + OverlappedPipe, +} +pub(super) struct File { + pub handle: Arc, + pub kind: Kind, + pipe: bool, +} +impl File { + pub fn duplicate(raw: HANDLE) -> Result { + ensure!(!raw.is_null() && raw != INVALID_HANDLE_VALUE, "missing terminal handle"); + let mut duplicate = null_mut(); + // SAFETY: duplicate this process's borrowed standard handle, non-inheritable. + unsafe { + check(DuplicateHandle( + GetCurrentProcess(), + raw, + GetCurrentProcess(), + &mut duplicate, + 0, + 0, + DUPLICATE_SAME_ACCESS, + ))?; + } + Self::owned(unsafe { OwnedHandle::from_raw_handle(duplicate) }) + } + pub fn owned(handle: OwnedHandle) -> Result { + let raw = handle.as_raw_handle(); + let file_type = unsafe { GetFileType(raw) }; + let mut mode = 0; + let kind = if unsafe { GetConsoleMode(raw, &mut mode) } != 0 { + Kind::Console + } else { + ensure!( + matches!(file_type, FILE_TYPE_DISK | FILE_TYPE_PIPE | FILE_TYPE_CHAR), + "unsupported terminal handle type" + ); + // This class-specific query follows .NET 10 SafeFileHandle.GetFileOptions: + // it returns the file object's mode directly and accepts only SUCCESS. + // Do not generalize this to other information classes or infer request + // completion by waiting on an inherited file's shared event. + // https://github.com/dotnet/runtime/blob/60629d14374c56f1cb51819049ad1fa529307f8d/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs#L182-L205 + let mut status: IO_STATUS_BLOCK = unsafe { std::mem::zeroed() }; + let mut info: FILE_MODE_INFORMATION = unsafe { std::mem::zeroed() }; + let result = unsafe { + NtQueryInformationFile( + raw, + &mut status, + (&mut info as *mut FILE_MODE_INFORMATION).cast(), + size_of::() as u32, + FileModeInformation, + ) + }; + ensure!(result == STATUS_SUCCESS, "cannot classify inherited terminal handle"); + if info.Mode & (FILE_SYNCHRONOUS_IO_ALERT | FILE_SYNCHRONOUS_IO_NONALERT) != 0 { + Kind::Synchronous + } else { + ensure!( + file_type == FILE_TYPE_PIPE, + "inherited asynchronous seekable files are unsupported for container exec" + ); + Kind::OverlappedPipe + } + }; + Ok(Self { + handle: Arc::new(handle), + kind, + pipe: file_type == FILE_TYPE_PIPE, + }) + } + pub fn raw(&self) -> HANDLE { + self.handle.as_raw_handle() + } + + pub fn read(&self, bytes: &mut [u8], shared: &Shared) -> Result { + if self.kind == Kind::Console { + bail!("console input requires Unicode decoding"); + } + loop { + match self.transfer(bytes.as_mut_ptr(), bytes.len(), false, shared)? { + // A zero-byte pipe message is not EOF. Only a broken pipe is. + Some(0) if self.pipe => continue, + Some(count) => return Ok(count), + None => return Ok(0), + } + } + } + pub fn write(&self, bytes: &[u8], shared: &Shared) -> Result { + Ok(self + .transfer(bytes.as_ptr().cast_mut(), bytes.len(), true, shared)? + .unwrap_or(0)) + } + fn transfer(&self, bytes: *mut u8, length: usize, write: bool, shared: &Shared) -> Result> { + ensure!(length <= super::MAX_DATA_BYTES, "terminal I/O exceeds bound"); + if shared.stopped() { + return Ok(None); + } + let mut count = 0; + let mut pending = if self.kind == Kind::OverlappedPipe { + Some(Pending::new(self.raw())?) + } else { + None + }; + let overlap = pending.as_mut().map_or(null_mut(), |value| &mut *value.overlap); + // SAFETY: the caller owns this bounded buffer until final completion; + // Pending retains its stable OVERLAPPED/event and drains on unwind. + let ok = unsafe { + if write { + WriteFile(self.raw(), bytes, length as u32, &mut count, overlap) + } else { + ReadFile(self.raw(), bytes, length as u32, &mut count, overlap) + } + }; + let mut error = if ok == 0 { + unsafe { GetLastError() } + } else { + ERROR_SUCCESS + }; + if error == ERROR_IO_PENDING { + let operation = pending.as_mut().expect("overlapped operation must own completion"); + operation.active = true; + // Cancellation may have arrived before ReadFile/WriteFile registered. + // The issuing owner rechecks it and cancels the exact operation. + loop { + if shared.stopped() { + operation.cancel(); + } + let done = unsafe { GetOverlappedResult(self.raw(), &*operation.overlap, &mut count, 0) }; + error = if done != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + if error != ERROR_IO_INCOMPLETE { + operation.active = false; + break; + } + operation.event.wait(20); + } + } + if shared.stopped() { + return Ok(None); + } + if !write && matches!(error, ERROR_BROKEN_PIPE | ERROR_HANDLE_EOF) { + return Ok(None); + } + ensure!( + error == ERROR_SUCCESS || (!write && error == ERROR_MORE_DATA), + "terminal I/O failed" + ); + ensure!(count as usize <= length, "invalid terminal I/O count"); + Ok(Some(count as usize)) + } + pub fn read_console(&self, units: &mut [u16], shared: &Shared) -> Result { + loop { + let mut count = 0; + if shared.stopped() { + return Ok(0); + } + let ok = unsafe { + SetLastError(ERROR_SUCCESS); + ReadConsoleW( + self.raw(), + units.as_mut_ptr().cast(), + units.len() as u32, + &mut count, + null(), + ) + }; + if shared.stopped() { + return Ok(0); + } + // Cooked console Ctrl+C/Break is forwarded by our control handler. + // It must interrupt and resume this read, not turn into EOF/failure. + if count == 0 && unsafe { GetLastError() } == ERROR_OPERATION_ABORTED { + continue; + } + check(ok)?; + ensure!(count as usize <= units.len(), "invalid console input count"); + return Ok(count as usize); + } + } +} + +struct Pending { + handle: HANDLE, + overlap: Box, + event: Event, + active: bool, +} +impl Pending { + fn new(handle: HANDLE) -> Result { + let event = Event::new()?; + let mut overlap: Box = Box::new(unsafe { std::mem::zeroed() }); + // The inherited file may belong to another owner's completion port. + // The low bit suppresses IOCP packets for this exact operation without + // changing the shared file's modes. Wait/close uses the unflagged event. + overlap.hEvent = event.raw().map_addr(|address| address | 1); + Ok(Self { + handle, + overlap, + event, + active: false, + }) + } + fn cancel(&self) { + unsafe { + CancelIoEx(self.handle, &*self.overlap); + } + } +} +impl Drop for Pending { + fn drop(&mut self) { + if self.active { + self.cancel(); + let mut count = 0; + // An unwind never abandons an accepted I/O operation or its memory. + unsafe { + GetOverlappedResult(self.handle, &*self.overlap, &mut count, 1); + } + } + } +} + +pub(super) struct ConsoleState { + files: Vec<(Arc, u32)>, + code_page: Option, + restored: bool, +} +impl ConsoleState { + pub fn prepare(files: &[Option; 3], tty: bool) -> Result { + let mut this = Self { + files: Vec::new(), + code_page: None, + restored: false, + }; + // Capture every alias before mutating any shared console buffer. + for file in files.iter().flatten().filter(|file| file.kind == Kind::Console) { + let mut mode = 0; + unsafe { + check(GetConsoleMode(file.raw(), &mut mode))?; + } + this.files.push((file.handle.clone(), mode)); + } + if files[1..].iter().flatten().any(|file| file.kind == Kind::Console) { + let original = unsafe { GetConsoleOutputCP() }; + ensure!(original != 0, "cannot read console output code page"); + this.code_page = Some(original); + unsafe { + check(SetConsoleOutputCP(65001))?; + } + } + for (index, file) in files.iter().enumerate().filter_map(|(i, f)| f.as_ref().map(|f| (i, f))) { + if file.kind != Kind::Console { + continue; + } + let mut mode = 0; + unsafe { + check(GetConsoleMode(file.raw(), &mut mode))?; + } + if index == 0 && tty { + mode = (mode | ENABLE_VIRTUAL_TERMINAL_INPUT | ENABLE_EXTENDED_FLAGS) + & !(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT | ENABLE_QUICK_EDIT_MODE); + } else if index != 0 { + mode |= ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING; + } + unsafe { + check(SetConsoleMode(file.raw(), mode))?; + } + } + Ok(this) + } + pub fn restore(&mut self) -> Result<()> { + if self.restored { + return Ok(()); + } + let mut failed = false; + for (file, mode) in &self.files { + failed |= unsafe { SetConsoleMode(file.as_raw_handle(), *mode) } == 0; + } + if let Some(code_page) = self.code_page { + failed |= unsafe { SetConsoleOutputCP(code_page) } == 0; + } + self.restored = !failed; + ensure!(!failed, "could not restore console settings"); + Ok(()) + } +} +impl Drop for ConsoleState { + fn drop(&mut self) { + let _ = self.restore(); + } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs new file mode 100644 index 00000000000..afe90e294ac --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs @@ -0,0 +1,471 @@ +// The process-wide console handler must stay exclusively owned across awaits. +#![allow(clippy::await_holding_lock)] +use super::*; +use std::{ + fs::OpenOptions, + os::windows::{ + fs::OpenOptionsExt, + io::{FromRawHandle, OwnedHandle}, + }, + ptr::{null, null_mut}, + time::Instant, +}; +use windows_sys::Win32::{ + Storage::FileSystem::*, + System::{Pipes::*, Threading::*}, +}; + +#[path = "console_tests.rs"] +mod console; + +// The real console child has a separate process. These pipe fixtures serialize +// only our removable process-wide Ctrl handler, never the user's console modes. +static TEST_OWNER: Mutex<()> = Mutex::new(()); + +fn finish(primary: std::thread::Result>, cleanup: impl IntoIterator>) -> Result { + let errors: Vec<_> = cleanup.into_iter().filter_map(Result::err).collect(); + let details = errors + .iter() + .map(|error| format!("{error:#}")) + .collect::>() + .join("; "); + match primary { + Ok(Ok(value)) if errors.is_empty() => Ok(value), + Ok(Ok(_)) => bail!("owned Windows fixture cleanup failed: {details}"), + Ok(Err(error)) if errors.is_empty() => Err(error), + Ok(Err(error)) => Err(error.context(format!("owned Windows fixture cleanup also failed: {details}"))), + Err(panic) => { + if !errors.is_empty() { + eprintln!("owned Windows fixture cleanup also failed: {details}"); + } + std::panic::resume_unwind(panic) + } + } +} +fn pipe() -> (OwnedHandle, OwnedHandle) { + let (mut read, mut write) = (null_mut(), null_mut()); + unsafe { + check(CreatePipe(&mut read, &mut write, null(), 4096)).unwrap(); + (OwnedHandle::from_raw_handle(read), OwnedHandle::from_raw_handle(write)) + } +} +fn fixture() -> (Terminal, Io, OwnedHandle, OwnedHandle, OwnedHandle) { + let (input, input_writer) = pipe(); + let (output_reader, output) = pipe(); + let (error_reader, error) = pipe(); + let (terminal, io) = Prepared::new( + [ + Some(File::owned(input).unwrap()), + Some(File::owned(output).unwrap()), + Some(File::owned(error).unwrap()), + ], + false, + ) + .unwrap() + .start() + .unwrap(); + (terminal, io, input_writer, output_reader, error_reader) +} +fn receive(handle: &OwnedHandle, size: usize) -> Vec { + let deadline = Instant::now() + Duration::from_secs(5); + let mut received = Vec::new(); + while received.len() < size { + let mut available = 0; + unsafe { + check(PeekNamedPipe( + handle.as_raw_handle(), + null_mut(), + 0, + null_mut(), + &mut available, + null_mut(), + )) + .unwrap(); + } + assert!(Instant::now() < deadline, "owned pipe output timed out"); + if available == 0 { + std::thread::sleep(Duration::from_millis(5)); + continue; + } + let mut bytes = vec![0; (available as usize).min(size - received.len())]; + let mut count = 0; + unsafe { + check(ReadFile( + handle.as_raw_handle(), + bytes.as_mut_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + )) + .unwrap(); + } + received.extend_from_slice(&bytes[..count as usize]); + } + received +} +fn write(handle: &OwnedHandle, bytes: &[u8]) { + let mut count = 0; + unsafe { + check(WriteFile( + handle.as_raw_handle(), + bytes.as_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + )) + .unwrap(); + } + assert_eq!(count as usize, bytes.len()); +} + +#[test] +fn native_windows_pipe_backpressure_cancellation_joins_both_workers() { + let _owner = lock(&TEST_OWNER); + let (mut terminal, io, _input_writer, _output_reader, _error_reader) = fixture(); + let (ack, mut completed) = oneshot::channel(); + io.output + .sender + .try_send(Output { + stream: OutputStream::Stdout, + bytes: vec![42; MAX_DATA_BYTES], + completed: ack, + }) + .unwrap(); + terminal.shared.wake.set(); + std::thread::sleep(Duration::from_millis(50)); + assert!(completed.try_recv().is_err()); + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(terminal.workers.is_empty()); + terminal.finish().unwrap(); + assert!(completed.blocking_recv().is_err()); +} + +#[test] +fn native_windows_cancel_before_io_registration_and_worker_panic_are_joined() { + let _owner = lock(&TEST_OWNER); + for panic in [false, true] { + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + terminal + .spawn("owned-race", move |shared| { + entered_tx.send(()).unwrap(); + while !shared.stopped() { + std::thread::yield_now(); + } + if panic { + panic!("owned terminal panic"); + } + // Intentionally issue a synchronous call after cancellation was + // requested. Repeated thread cancellation must catch registration. + let (read, _write) = pipe(); + let mut byte = 0; + let mut count = 0; + unsafe { + ReadFile(read.as_raw_handle(), &mut byte, 1, &mut count, null_mut()); + } + Ok(()) + }) + .unwrap(); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + let result = terminal.finish(); + assert_eq!(result.is_err(), panic); + assert!(terminal.workers.is_empty()); + } +} + +#[tokio::test] +async fn native_windows_pipe_bytes_eof_and_signal_numbers() { + let _owner = lock(&TEST_OWNER); + let (mut terminal, mut io, writer, reader, error_reader) = fixture(); + write(&writer, &[0, 255, 1, 13, 10]); + drop(writer); + let Input::Data(bytes) = io.input.recv().await.unwrap() else { + panic!("missing input bytes") + }; + assert_eq!(bytes, [0, 255, 1, 13, 10]); + assert!(matches!(io.input.recv().await, Some(Input::Eof))); + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![255, 0, 13, 10], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + written.await.unwrap().unwrap(); + assert_eq!(receive(&reader, 4), [255, 0, 13, 10]); + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stderr, + bytes: vec![0, 254, 128], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + written.await.unwrap().unwrap(); + assert_eq!(receive(&error_reader, 3), [0, 254, 128]); + let mut signals = terminal.signals().unwrap(); + assert_eq!(control::invoke(CTRL_C_EVENT), 1); + assert!(matches!(signals.next().await, Some(Ok(ClientControl::Signal(2))))); + assert_eq!(control::invoke(CTRL_BREAK_EVENT), 1); + assert!(matches!(signals.next().await, Some(Ok(ClientControl::Signal(3))))); + terminal.finish().unwrap(); +} + +#[test] +fn native_windows_synchronous_file_offset_and_asynchronous_file_rejection() { + let _owner = lock(&TEST_OWNER); + use std::io::{Read, Seek, SeekFrom, Write}; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("owned-redirection"); + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .unwrap(); + file.write_all(b"abcdef").unwrap(); + file.seek(SeekFrom::Start(2)).unwrap(); + let classified = File::duplicate(file.as_raw_handle()).unwrap(); + assert_eq!(classified.kind, Kind::Synchronous); + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let mut bytes = [0; 2]; + assert_eq!(classified.read(&mut bytes, &terminal.shared).unwrap(), 2); + assert_eq!(&bytes, b"cd"); + let mut remainder = Vec::new(); + file.read_to_end(&mut remainder).unwrap(); + assert_eq!(&remainder, b"ef"); + let asynchronous = OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OVERLAPPED) + .open(path) + .unwrap(); + assert!(File::duplicate(asynchronous.as_raw_handle()) + .err() + .unwrap() + .to_string() + .contains("asynchronous seekable")); + terminal.finish().unwrap(); +} + +fn overlapped_pipe(output: bool) -> (OwnedHandle, OwnedHandle) { + let unique = tempfile::tempdir().unwrap(); + let name: Vec = format!( + r"\\.\pipe\spacetimedb-exec-{}-{}", + std::process::id(), + unique.path().file_name().unwrap().to_string_lossy() + ) + .encode_utf16() + .chain([0]) + .collect(); + let access = if output { + PIPE_ACCESS_OUTBOUND + } else { + PIPE_ACCESS_INBOUND + }; + let raw = unsafe { + CreateNamedPipeW( + name.as_ptr(), + access | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 4096, + 4096, + 0, + null(), + ) + }; + assert_ne!(raw, INVALID_HANDLE_VALUE); + let server = unsafe { OwnedHandle::from_raw_handle(raw) }; + let access = if output { GENERIC_READ } else { GENERIC_WRITE }; + let raw = unsafe { CreateFileW(name.as_ptr(), access, 0, null(), OPEN_EXISTING, 0, null_mut()) }; + assert_ne!(raw, INVALID_HANDLE_VALUE); + let client = unsafe { OwnedHandle::from_raw_handle(raw) }; + let mut connected: windows_sys::Win32::System::IO::OVERLAPPED = unsafe { std::mem::zeroed() }; + // The exact client has already connected; no asynchronous accept remains. + assert_eq!(unsafe { ConnectNamedPipe(server.as_raw_handle(), &mut connected) }, 0); + assert_eq!(unsafe { GetLastError() }, ERROR_PIPE_CONNECTED); + (server, client) +} + +#[tokio::test] +async fn native_windows_overlapped_pipe_cancel_observes_exact_completion() { + let _owner = lock(&TEST_OWNER); + for output in [false, true] { + let (server, client) = overlapped_pipe(output); + let file = File::owned(server).unwrap(); + assert_eq!(file.kind, Kind::OverlappedPipe); + let (unused_reader, ordinary_output) = pipe(); + let files = if output { + [None, Some(file), None] + } else { + [Some(file), Some(File::owned(ordinary_output).unwrap()), None] + }; + let (mut terminal, mut io) = Prepared::new(files, false).unwrap().start().unwrap(); + let mut ack = None; + if output { + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![0x81; MAX_DATA_BYTES], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + ack = Some(written); + } else { + write(&client, &[]); + write(&client, &[0, 255, 13, 10]); + let Input::Data(bytes) = tokio::time::timeout(Duration::from_secs(2), io.input.recv()) + .await + .unwrap() + .unwrap() + else { + panic!("zero pipe message became EOF") + }; + assert_eq!(bytes, [0, 255, 13, 10]); + } + tokio::time::sleep(Duration::from_millis(30)).await; + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(terminal.workers.is_empty()); + if let Some(written) = ack { + assert!(written.await.is_err()); + } + drop(unused_reader); + } +} + +#[tokio::test] +async fn native_windows_close_before_ready_restores_then_releases_callback() { + let _owner = lock(&TEST_OWNER); + use futures::{FutureExt, StreamExt}; + let (mut terminal, mut io, _writer, _reader, _error_reader) = fixture(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + spacetimedb_lib::Identity::ZERO.to_hex() + ); + let (started, start) = std::sync::mpsc::channel(); + let callback = std::thread::spawn(move || { + start.recv_timeout(Duration::from_secs(5)).unwrap(); + control::invoke(CTRL_CLOSE_EVENT) + }); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async( + socket, + |_request: &tokio_tungstenite::tungstenite::handshake::server::Request, + mut response: tokio_tungstenite::tungstenite::handshake::server::Response| { + response.headers_mut().insert( + "sec-websocket-protocol", + spacetimedb_lib::container::exec::SUBPROTOCOL.parse().unwrap(), + ); + Ok(response) + }, + ) + .await + .unwrap(); + assert!(socket.next().await.is_some()); + started.send(()).unwrap(); + assert!(socket.next().await.is_none_or(|value| value.is_err())); + }); + let start = spacetimedb_lib::container::exec::ExecStart { + generation: 1, + argv: vec!["literal-command".into()], + working_directory: None, + environment: Default::default(), + stdin: true, + terminal: None, + }; + let result = std::panic::AssertUnwindSafe(async { + let result = tokio::time::timeout( + Duration::from_secs(3), + super::super::session::run( + origin.parse().unwrap(), + "Bearer owned-loopback-fixture".parse().unwrap(), + spacetimedb_lib::Identity::ZERO, + start, + &mut io, + futures::stream::pending(), + ), + ) + .await?; + ensure!(result.is_err(), "close was reported as a guest exit"); + Ok(()) + }) + .catch_unwind() + .await; + // Join and release the handler even if the assertion below will fail. + let terminal_result = terminal.finish(); + let callback_result = callback + .join() + .map_err(|_| anyhow::anyhow!("close callback panicked")) + .and_then(|result| { + ensure!(result == 1, "close callback was not handled"); + Ok(()) + }); + let server_result = server.await.context("owned close fixture server failed"); + finish(result, [terminal_result, callback_result, server_result]).unwrap(); + assert_eq!(*lock(&terminal.shared.callbacks), 0); +} + +#[test] +fn native_windows_console_unicode_surrogates_are_not_split() { + let mut pending = None; + assert_eq!(decode_console(&[0xd83e], &mut pending).unwrap(), ""); + assert_eq!( + decode_console(&[0xdd80, 27, 91, 65], &mut pending).unwrap(), + "🦀\u{1b}[A" + ); + assert!(decode_console(&[0xdc00], &mut pending).is_err()); +} + +#[test] +fn native_windows_overlapped_pipe_suppresses_inherited_completion_port_packets() { + let _owner = lock(&TEST_OWNER); + use windows_sys::Win32::System::IO::{CreateIoCompletionPort, GetQueuedCompletionStatus}; + for output in [false, true] { + let (server, client) = overlapped_pipe(output); + let raw = unsafe { CreateIoCompletionPort(server.as_raw_handle(), null_mut(), 123, 1) }; + assert!(!raw.is_null()); + let port = unsafe { OwnedHandle::from_raw_handle(raw) }; + // Dup preserves the existing IOCP association. No shared completion mode + // is changed by classification or the adapter's per-operation events. + let file = File::duplicate(server.as_raw_handle()).unwrap(); + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let primary = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if output { + assert_eq!(file.write(&[0, 255, 7], &terminal.shared)?, 3); + assert_eq!(receive(&client, 3), [0, 255, 7]); + } else { + write(&client, &[0, 255, 8]); + let mut bytes = [0; 3]; + assert_eq!(file.read(&mut bytes, &terminal.shared)?, 3); + assert_eq!(bytes, [0, 255, 8]); + } + let (mut bytes, mut key, mut overlap) = (0, 0, null_mut()); + let result = + unsafe { GetQueuedCompletionStatus(port.as_raw_handle(), &mut bytes, &mut key, &mut overlap, 50) }; + assert_eq!(result, 0); + assert_eq!(unsafe { GetLastError() }, WAIT_TIMEOUT); + assert!(overlap.is_null(), "adapter operation escaped to inherited IOCP"); + Ok(()) + })); + let cleanup = terminal.finish(); + finish(primary, [cleanup]).unwrap(); + } +} diff --git a/crates/cli/src/subcommands/container/exec/tests.rs b/crates/cli/src/subcommands/container/exec/tests.rs new file mode 100644 index 00000000000..356f92ac528 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/tests.rs @@ -0,0 +1,490 @@ +use super::session::{Input, Io, OutputSender}; +use super::*; +use futures::{SinkExt, StreamExt}; +use reqwest::{header::HeaderValue, Url}; +use spacetimedb_lib::{ + container::exec::{self as wire, ClientControl, ExecReady, ServerControl}, + Identity, Uuid, +}; +use tokio::{ + net::TcpListener, + sync::{mpsc, oneshot}, +}; +use tokio_tungstenite::tungstenite::{ + handshake::server::{Request, Response}, + Message, +}; + +fn arguments(values: &[&str]) -> Result { + cli().try_get_matches_from(values) +} +#[test] +fn literal_arguments_and_environment_are_bounded_and_redacted() { + cli().help_expected(true).debug_assert(); + let args = arguments(&[ + "exec", + "db", + "-i", + "--workdir", + "/a b", + "-e", + "UNICODE=🦀=literal", + "--", + "/bin/echo", + "$(no-shell)", + "--flag", + ]) + .unwrap(); + let value = start(&args, 42).unwrap(); + assert_eq!(value.argv, ["/bin/echo", "$(no-shell)", "--flag"]); + assert_eq!(value.environment["UNICODE"], "🦀=literal"); + assert_eq!(value.working_directory.as_deref(), Some("/a b")); + for invalid in ["SPACETIMEDB_TOKEN=secret-value", "BAD-NAME=secret-value", "NO_EQUALS"] { + let args = arguments(&["exec", "db", "-e", invalid, "--", "true"]).unwrap(); + let error = start(&args, 1).unwrap_err().to_string(); + assert!(!error.contains("secret-value")); + } + let args = arguments(&["exec", "db", "-e", "A=1", "-e", "A=2", "--", "true"]).unwrap(); + assert!(start(&args, 1).is_err()); + assert!(arguments(&["exec", "db", "-t", "--", "true"]).is_err()); + let args = arguments(&["exec", "db", "--workdir", "relative", "--", "true"]).unwrap(); + assert!(start(&args, 1).is_err()); +} + +#[tokio::test] +async fn real_loopback_socket_preserves_binary_eof_controls_and_exit() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + ); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async(stream, |request: &Request, mut response: Response| { + assert_eq!(request.uri().query(), Some("generation=42")); + assert_eq!(request.headers()["authorization"], "Bearer owned-loopback-fixture"); + assert_eq!(request.headers()["sec-websocket-protocol"], wire::SUBPROTOCOL); + assert!(!request.uri().to_string().contains("owned-loopback-fixture")); + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + Ok(response) + }) + .await + .unwrap(); + let first = socket.next().await.unwrap().unwrap(); + let ClientControl::Start(start) = ClientControl::decode(first.into_data().as_ref()).unwrap() else { + panic!("missing Start") + }; + assert_eq!(start.argv, ["/bin/echo", "literal"]); + let ready = ServerControl::Ready(ExecReady { + database_identity: Identity::ZERO, + generation: 42, + session_id: Uuid::from_u128(7), + tty: false, + }); + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + let mut data = false; + let mut eof = false; + let mut signal = false; + let mut resize = false; + while !(data && eof && signal && resize) { + match socket.next().await.unwrap().unwrap() { + Message::Binary(value) => { + assert_eq!(wire::decode_stdin(&value).unwrap(), &[0, 255, 1]); + data = true; + } + Message::Text(value) => match ClientControl::decode(value.as_bytes()).unwrap() { + ClientControl::StdinEof => { + assert!(!eof); + eof = true; + } + ClientControl::Signal(10) => signal = true, + ClientControl::Resize(wire::TerminalSize { rows: 24, columns: 80 }) => resize = true, + _ => panic!("unexpected control"), + }, + _ => panic!("unexpected frame"), + } + } + for stream in [wire::Stream::Stdout, wire::Stream::Stderr] { + socket + .send(Message::Binary( + wire::encode_data(stream, &[255, 0, 128]).unwrap().into(), + )) + .await + .unwrap(); + } + socket + .send(Message::Text( + serde_json::to_string(&ServerControl::Exit { exit_code: 37 }) + .unwrap() + .into(), + )) + .await + .unwrap(); + }); + let (input_tx, input) = mpsc::channel(2); + input_tx.send(Input::Data(vec![0, 255, 1])).await.unwrap(); + input_tx.send(Input::Eof).await.unwrap(); + let (output_tx, mut output) = mpsc::channel::(1); + let (_completed, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let options = start( + &arguments(&["exec", "db", "-i", "--", "/bin/echo", "literal"]).unwrap(), + 42, + ) + .unwrap(); + let signals = futures::stream::iter([ + Ok(ClientControl::Signal(10)), + Ok(ClientControl::Resize(wire::TerminalSize { rows: 24, columns: 80 })), + ]) + .chain(futures::stream::pending()); + let client = session::run( + Url::parse(&origin).unwrap(), + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + signals, + ); + let output_reader = async { + let mut seen = Vec::new(); + for _ in 0..2 { + let value = output.recv().await.unwrap(); + assert_eq!(value.bytes, [255, 0, 128]); + seen.push(value.stream); + value.completed.send(Ok(())).unwrap(); + } + assert_eq!(seen, [wire::Stream::Stdout, wire::Stream::Stderr]); + }; + let results = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!(client, output_reader) + }) + .await; + if results.is_err() { + server.abort(); + } + let joined = server.await; + assert!(joined.is_ok()); + assert_eq!(results.unwrap().0.unwrap(), 37); +} + +#[test] +fn status_requires_the_exact_running_generation_without_http_readiness() { + use spacetimedb_lib::container::operations::*; + let mut status = ContainerStatus { + database_identity: Identity::ZERO, + published: true, + configuration: None, + endpoints: EndpointStatus::Pending, + operational: Some(OperationalState { + desired_revision: spacetimedb_lib::Hash::from_byte_array([0; 32]), + desired_state: DesiredState::Running, + generation: 42, + condition: Condition::None, + restart_pending: false, + restart_attempt: 0, + restart_not_before_ms: 0, + current_instance: Some(CurrentInstance { + generation: 42, + state: ObservedState::Running, + observed_revision: None, + applied_env_generation: None, + exit_code: None, + oom_killed: false, + usage: None, + condition: Condition::None, + }), + }), + }; + assert_eq!(running_generation(&status).unwrap(), 42); + for state in [ + ObservedState::Pending, + ObservedState::Starting, + ObservedState::Draining, + ObservedState::Stopped, + ObservedState::Completed, + ObservedState::Failed, + ] { + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .state = state; + assert!(running_generation(&status).is_err()); + } + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .state = ObservedState::Ready; + assert_eq!(running_generation(&status).unwrap(), 42); + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .generation = 41; + assert!(running_generation(&status).is_err()); + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .generation = 42; + status.operational.as_mut().unwrap().desired_state = DesiredState::Stopped; + assert!(running_generation(&status).is_err()); +} + +#[tokio::test] +async fn real_loopback_rejects_bad_bindings_order_and_unobserved_exit_without_replay() { + for case in 0..8 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = Url::parse(&format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + )) + .unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async(stream, move |_: &Request, mut response: Response| { + if case != 0 { + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + } + Ok(response) + }) + .await + .unwrap(); + if case != 0 { + let first = socket.next().await.unwrap().unwrap(); + assert!(matches!( + ClientControl::decode(first.into_data().as_ref()).unwrap(), + ClientControl::Start(_) + )); + let ready = ServerControl::Ready(ExecReady { + database_identity: if case == 1 { + Identity::from_byte_array([1; 32]) + } else { + Identity::ZERO + }, + generation: if case == 2 { 43 } else { 42 }, + session_id: Uuid::from_u128(if case == 3 { 0 } else { 7 }), + tty: case == 4, + }); + if case == 5 { + socket + .send(Message::Binary( + wire::encode_data(wire::Stream::Stdout, b"premature").unwrap().into(), + )) + .await + .unwrap(); + } else { + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + if case == 6 { + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + } + if case == 7 { + socket.close(None).await.unwrap(); + } + } + } + // The client must close this exact connection and never dial a retry. + let closed = tokio::time::timeout(std::time::Duration::from_secs(2), socket.next()).await; + assert!(matches!(closed, Ok(None | Some(Err(_)) | Some(Ok(Message::Close(_)))))); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), listener.accept()) + .await + .is_err() + ); + }); + let (_input_tx, input) = mpsc::channel(1); + let (output_tx, _output) = mpsc::channel(1); + let (_completed, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let options = start(&arguments(&["exec", "db", "--", "true"]).unwrap(), 42).unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(4), + session::run( + url, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + futures::stream::pending(), + ), + ) + .await; + if result.is_err() { + server.abort(); + } + let joined = server.await; + assert!(joined.is_ok(), "case {case}"); + assert!(result.unwrap().is_err(), "case {case}"); + } +} + +#[tokio::test] +async fn status_uses_authenticated_explicit_loopback_and_pins_the_resolved_identity() { + use spacetimedb_lib::container::operations::EndpointStatus; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = Url::parse(&format!("http://{}", listener.local_addr().unwrap())).unwrap(); + let identity = Identity::from_byte_array([7; 32]); + let status = ContainerStatus { + database_identity: identity, + published: true, + configuration: None, + operational: None, + endpoints: EndpointStatus::Pending, + }; + let router = axum::Router::new().route( + "/v1/database/owned-test-db/container/status", + axum::routing::get(move |headers: axum::http::HeaderMap| async move { + assert_eq!(headers["authorization"], "Bearer owned-loopback-fixture"); + axum::Json(status) + }), + ); + let (stop, stopped) = oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + let client = super::super::operations::ContainerClient::new( + origin, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + ) + .unwrap(); + let result = client.status("owned-test-db").await; + let _ = stop.send(()); + let joined = server.await.unwrap(); + joined.unwrap(); + let status = result.unwrap(); + assert_eq!(status.database_identity, identity); + let endpoint = client.url(status.database_identity.to_hex().as_ref(), "exec").unwrap(); + assert_eq!( + endpoint.path(), + format!("/v1/database/{}/container/exec", identity.to_hex()) + ); + assert!(running_generation(&status).is_err()); +} + +#[tokio::test] +async fn terminal_cancellation_closes_pending_handshake_and_ready_without_replay() { + use tokio::io::AsyncReadExt; + for handshake in [true, false] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = Url::parse(&format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + )) + .unwrap(); + let (_input_tx, input) = mpsc::channel(1); + let (output_tx, _output) = mpsc::channel(1); + let (cancel, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let mut server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + if handshake { + let mut byte = [0]; + assert_eq!(socket.read(&mut byte).await.unwrap(), 1); + let _ = cancel.send(Err(anyhow::anyhow!("owned terminal cancellation"))); + let mut pending_request = Vec::new(); + socket.read_to_end(&mut pending_request).await.unwrap(); + } else { + let mut socket = + tokio_tungstenite::accept_hdr_async(socket, |_request: &Request, mut response: Response| { + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + Ok(response) + }) + .await + .unwrap(); + assert!(matches!(socket.next().await, Some(Ok(Message::Text(_))))); + let _ = cancel.send(Err(anyhow::anyhow!("owned terminal cancellation"))); + assert!(matches!( + socket.next().await, + None | Some(Err(_)) | Some(Ok(Message::Close(_))) + )); + } + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), listener.accept()) + .await + .is_err() + ); + }); + let options = start(&arguments(&["exec", "db", "--", "true"]).unwrap(), 42).unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + session::run( + url, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + futures::stream::pending(), + ), + ) + .await; + let joined = tokio::time::timeout(std::time::Duration::from_secs(2), &mut server).await; + let forced = joined.is_err(); + if forced { + server.abort(); + let _ = server.await; + } + assert!(!forced, "owned cancellation server failed to stop"); + joined.unwrap().unwrap(); + assert!(result + .unwrap() + .unwrap_err() + .to_string() + .contains("owned terminal cancellation")); + } +} diff --git a/crates/cli/src/subcommands/container/logs.rs b/crates/cli/src/subcommands/container/logs.rs new file mode 100644 index 00000000000..5160eed9f8c --- /dev/null +++ b/crates/cli/src/subcommands/container/logs.rs @@ -0,0 +1,262 @@ +//! Authorized, bounded log pages pinned to one capture across requests. + +use super::operations::ContainerClient; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{header::AUTHORIZATION, StatusCode}; +use spacetimedb_lib::{ + container::{ + logs::*, + operations::{ContainerApiError, ContainerErrorCode}, + }, + Hash, Identity, Uuid, +}; +use std::{io::Write, time::Duration}; + +pub(super) fn cli() -> Command { + Command::new("logs") + .about("Read retained stdout and stderr from one container attempt") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg(Arg::new("generation").long("generation").value_parser(clap::value_parser!(u64).range(1..)) + .help("Read this attempt; omitted selects the current generation once")) + .arg(Arg::new("cursor").long("cursor").requires("generation") + .help("Resume after an opaque cursor returned by --json")) + .arg(Arg::new("follow").long("follow").short('f').action(ArgAction::SetTrue) + .help("Wait for further output from the selected attempt until it ends")) + .arg(Arg::new("json").long("json").action(ArgAction::SetTrue) + .help("Print one JSON page per line, including timestamps, streams, and resume cursors")) + .after_help("A restart does not change the selected attempt. Without --json, stdout and stderr retain their original bytes and streams. Retention gaps and interrupted capture are reported on stderr.") +} + +pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let database = args.get_one::("database").context("database is required")?; + let mut query = ContainerLogQuery { + generation: args.get_one::("generation").copied(), + cursor: args.get_one::("cursor").cloned(), + follow: args.get_flag("follow"), + }; + query.validate().map_err(anyhow::Error::msg)?; + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new(server, auth.to_header().context("container logs require a login")?)?; + // Names are resolved only once. All continuation requests use the Identity + // and exact generation/capture returned by the first authorized page. + let mut target = database.clone(); + let mut reader = Selection::new(database.parse().ok(), &query); + let json = args.get_flag("json"); + let mut reported_loss = None; + loop { + let page = tokio::select! { + result = fetch(&client, &target, &query) => result?, + signal = tokio::signal::ctrl_c() => { signal?; return Ok(()); } + }; + reader.accept(&page)?; + { + let mut out = std::io::stdout().lock(); + let mut err = std::io::stderr().lock(); + write_page(&page, json, &mut out, &mut err, &mut reported_loss)?; + } + if let Some(end) = page.end { + if end != LogEnd::Eof || page.loss.is_some() { + bail!("container log capture ended with incomplete output; inspect --json for the recorded reason"); + } + return Ok(()); + } + if !query.follow && !page.has_more { + return Ok(()); + } + target = page.database_identity.to_hex().to_string(); + query.generation = Some(page.generation); + query.cursor = Some(page.next_cursor); + // A healthy follow request long-polls. Bound retries even if a server + // repeatedly returns an immediate heartbeat without new output. + if page.records.is_empty() { + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis(200)) => {}, + signal = tokio::signal::ctrl_c() => { signal?; return Ok(()); } + } + } + } +} + +async fn fetch(client: &ContainerClient, database: &str, query: &ContainerLogQuery) -> Result { + query.validate().map_err(anyhow::Error::msg)?; + let mut response = client + .http + .get(client.url(database, "logs")?) + .header(AUTHORIZATION, client.authorization.clone()) + .query(query) + .send() + .await + .map_err(|_| anyhow::anyhow!("container logs could not be reached on the selected server"))?; + let status = response.status(); + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_LOG_PAGE_BYTES as u64), + "container log response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container log response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_LOG_PAGE_BYTES.saturating_sub(bytes.len()), + "container log response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != StatusCode::OK { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit reading container logs", + Some(ContainerErrorCode::NotFound) => "the selected container log history was not found or has expired", + Some(ContainerErrorCode::InvalidRequest) => "invalid container log selection or cursor", + Some(ContainerErrorCode::Conflict) => "the selected container log capture is no longer available", + _ => "container log service is unavailable on the selected server", + }, + status.as_u16() + ); + } + let page: ContainerLogPage = + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container log response"))?; + page.validate().map_err(anyhow::Error::msg)?; + Ok(page) +} + +#[derive(PartialEq, Eq)] +struct Capture { + identity: Identity, + generation: u64, + revision: Hash, + publication: Uuid, + epoch: u64, + id: Uuid, +} +struct Selection { + identity: Option, + generation: Option, + capture: Option, + cursor: Option, + sequence: Option, + loss: Option, +} +impl Selection { + fn new(identity: Option, query: &ContainerLogQuery) -> Self { + Self { + identity, + generation: query.generation, + capture: None, + cursor: query.cursor.clone(), + sequence: None, + loss: None, + } + } + fn accept(&mut self, page: &ContainerLogPage) -> Result<()> { + page.validate().map_err(anyhow::Error::msg)?; + ensure!( + self.identity.is_none_or(|identity| identity == page.database_identity) + && self.generation.is_none_or(|generation| generation == page.generation), + "container logs returned another database or generation" + ); + ensure!( + self.loss.is_none_or(|loss| page.loss == Some(loss)), + "container logs changed the recorded capture loss" + ); + let capture = Capture { + identity: page.database_identity, + generation: page.generation, + revision: page.deployment_revision, + publication: page.publication_operation, + epoch: page.publication_epoch, + id: page.capture_id, + }; + ensure!( + self.capture.as_ref().is_none_or(|expected| expected == &capture), + "container logs changed the selected capture" + ); + let mut sequence = self.sequence; + for record in &page.records { + if let Some(previous) = sequence { + ensure!( + record.sequence > previous, + "container logs repeated or reordered records" + ); + ensure!( + page.retention_gap || previous.checked_add(1) == Some(record.sequence), + "container logs omitted records without a retention notice" + ); + } + sequence = Some(record.sequence); + } + ensure!( + page.records.is_empty() || self.cursor.as_ref() != Some(&page.next_cursor), + "container logs did not advance their cursor" + ); + self.identity = Some(page.database_identity); + self.generation = Some(page.generation); + self.capture = Some(capture); + self.cursor = Some(page.next_cursor.clone()); + self.sequence = sequence; + self.loss = page.loss; + Ok(()) + } +} + +fn write_page( + page: &ContainerLogPage, + json: bool, + out: &mut impl Write, + err: &mut impl Write, + reported_loss: &mut Option, +) -> Result<()> { + if json { + serde_json::to_writer(&mut *out, page)?; + out.write_all(b"\n")?; + } else { + if page.retention_gap { + writeln!(err, "[container logs: earlier records were removed by retention]")?; + } + for record in &page.records { + match &record.event { + LogEvent::Data { + stream: LogStream::Stdout, + bytes, + .. + } => out.write_all(bytes)?, + LogEvent::Data { + stream: LogStream::Stderr, + bytes, + .. + } => err.write_all(bytes)?, + LogEvent::Gap { + reason, missed_bytes, .. + } => writeln!( + err, + "[container logs: capture gap {reason:?}, missing bytes {missed_bytes:?}]" + )?, + } + } + if let Some(loss) = page.loss + && Some(loss) != *reported_loss + { + writeln!(err, "[container logs: incomplete capture, {loss:?}]")?; + } + } + *reported_loss = page.loss; + out.flush()?; + err.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/logs/tests.rs b/crates/cli/src/subcommands/container/logs/tests.rs new file mode 100644 index 00000000000..73d2570bdc6 --- /dev/null +++ b/crates/cli/src/subcommands/container/logs/tests.rs @@ -0,0 +1,261 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::{Path, Query}, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn page() -> ContainerLogPage { + ContainerLogPage { + database_identity: Identity::ONE, + generation: 7, + deployment_revision: Hash::from_byte_array([3; 32]), + publication_operation: Uuid::from_u128(1), + publication_epoch: 2, + capture_id: Uuid::from_u128(3), + records: vec![ + LogRecord { + sequence: 1, + event: LogEvent::Data { + timestamp_micros: 1, + stream: LogStream::Stdout, + bytes: vec![0, 255, b'a'], + }, + }, + LogRecord { + sequence: 2, + event: LogEvent::Data { + timestamp_micros: 2, + stream: LogStream::Stderr, + bytes: vec![254, b'\n'], + }, + }, + ], + next_cursor: "cursor_2".into(), + oldest_retained_sequence: 1, + retention_gap: false, + has_more: false, + end: None, + loss: None, + } +} + +#[test] +fn output_preserves_bytes_streams_and_structured_metadata() -> Result<()> { + let page = page(); + let (mut out, mut err) = (vec![], vec![]); + write_page(&page, false, &mut out, &mut err, &mut None)?; + assert_eq!(out, [0, 255, b'a']); + assert_eq!(err, [254, b'\n']); + out.clear(); + err.clear(); + write_page(&page, true, &mut out, &mut err, &mut None)?; + assert!(err.is_empty()); + let decoded: ContainerLogPage = serde_json::from_slice(&out)?; + assert!(decoded == page); + assert_eq!(out.last(), Some(&b'\n')); + Ok(()) +} + +#[test] +fn continuation_pins_capture_and_rejects_replays_or_silent_gaps() -> Result<()> { + let mut selection = Selection::new(Some(Identity::ONE), &ContainerLogQuery::default()); + selection.accept(&page())?; + let mut next = page(); + next.records.clear(); + selection.accept(&next)?; // Empty heartbeat can retain the same cursor. + for changed in 0..7 { + let mut different = next.clone(); + match changed { + 0 => different.database_identity = Identity::ZERO, + 1 => different.generation += 1, + 2 => different.capture_id = Uuid::from_u128(4), + 3 => different.deployment_revision = Hash::from_byte_array([4; 32]), + 4 => different.publication_epoch += 1, + 5 => different.publication_operation = Uuid::from_u128(5), + _ => different.records = page().records, + } + assert!(selection.accept(&different).is_err()); + } + next.records.push(LogRecord { + sequence: 4, + event: LogEvent::Data { + timestamp_micros: 3, + stream: LogStream::Stdout, + bytes: b"last".to_vec(), + }, + }); + next.next_cursor = "cursor_4".into(); + assert!(selection.accept(&next).is_err()); + next.retention_gap = true; + next.oldest_retained_sequence = 4; + next.end = Some(LogEnd::Eof); + next.loss = Some(LogLoss::DrainTimeout); + selection.accept(&next)?; + next.records.clear(); + next.loss = None; + assert!(selection.accept(&next).is_err()); + Ok(()) +} + +#[test] +fn incomplete_capture_and_retention_are_visible_without_repeating_loss() -> Result<()> { + let mut page = page(); + page.records.clear(); + page.retention_gap = true; + page.loss = Some(LogLoss::DrainTimeout); + page.end = Some(LogEnd::Eof); + let (mut out, mut err, mut reported) = (vec![], vec![], None); + write_page(&page, false, &mut out, &mut err, &mut reported)?; + let text = String::from_utf8(err.clone())?; + assert!(text.contains("removed by retention")); + assert!(text.contains("incomplete capture, DrainTimeout")); + assert!(out.is_empty()); + page.retention_gap = false; + err.clear(); + write_page(&page, false, &mut out, &mut err, &mut reported)?; + assert!(err.is_empty()); + Ok(()) +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn requests_preserve_selection_and_bound_untrusted_responses() -> Result<()> { + let count = Arc::new(AtomicUsize::new(0)); + let requests = count.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/logs", + get( + move |Path(database): Path, Query(query): Query, headers: HeaderMap| { + let requests = requests.clone(); + async move { + requests.fetch_add(1, Ordering::SeqCst); + assert_eq!(headers.get(header::AUTHORIZATION).unwrap(), "Bearer disposable-test"); + assert!(!headers.contains_key(header::COOKIE)); + assert_eq!(query.generation, Some(7)); + assert_eq!(query.cursor.as_deref(), Some("cursor_0")); + assert!(query.follow); + match database.as_str() { + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_LOG_PAGE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "denied" => ( + StatusCode::FORBIDDEN, + Json(ContainerApiError { + error: ContainerErrorCode::AccessDenied, + }), + ) + .into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "project/child?query#fragment" => Json(page()).into_response(), + _ => StatusCode::NOT_FOUND.into_response(), + } + } + }, + ), + )) + .await?; + let result = async { + let client = ContainerClient::new(reqwest::Url::parse(&server.origin)?, "Bearer disposable-test".parse()?)?; + let query = ContainerLogQuery { + generation: Some(7), + cursor: Some("cursor_0".into()), + follow: true, + }; + let actual = fetch(&client, "project/child?query#fragment", &query).await?; + ensure!(actual == page(), "received different page"); + for (database, expected) in [ + ("oversize", "size limit"), + ("redirect", "HTTP 302"), + ("denied", "does not permit"), + ("invalid", "invalid container log response"), + ] { + let error = match fetch(&client, database, &query).await { + Ok(_) => bail!("expected rejection"), + Err(error) => error.to_string(), + }; + ensure!( + error.contains(expected) && !error.contains("private diagnostic"), + "wrong fixed error" + ); + } + ensure!(count.load(Ordering::SeqCst) == 5, "unexpected redirected request"); + Ok(()) + } + .await; + let shutdown = server.shutdown().await; + result.and(shutdown) +} + +#[test] +fn command_requires_generation_for_cursor_and_parses_explicit_server() { + assert!(cli() + .try_get_matches_from(["logs", "demo", "--cursor", "resume"]) + .is_err()); + assert!(cli() + .try_get_matches_from(["logs", "demo", "--generation", "0"]) + .is_err()); + let args = cli() + .try_get_matches_from([ + "logs", + "demo", + "--server", + "http://127.0.0.1:3000", + "--generation", + "7", + "--cursor", + "resume", + "--follow", + "--json", + ]) + .unwrap(); + assert_eq!(args.get_one::("generation"), Some(&7)); + assert_eq!( + args.get_one::("server").map(String::as_str), + Some("http://127.0.0.1:3000") + ); + assert!(args.get_flag("follow") && args.get_flag("json")); +} diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs new file mode 100644 index 00000000000..4381500f93c --- /dev/null +++ b/crates/cli/src/subcommands/container/operations.rs @@ -0,0 +1,344 @@ +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION}, + Client, Method, StatusCode, Url, +}; +use serde::de::DeserializeOwned; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, operations::*}, + Identity, Uuid, +}; +use std::time::Duration; + +const MAX_RESPONSE: usize = 64 * 1024; +pub(super) fn commands(command: Command) -> Command { + let base = |name: &'static str, about: &'static str| { + Command::new(name) + .about(about) + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print the typed response as JSON"), + ) + }; + let action = |name, about| { + base(name, about) + .arg(Arg::new("request_id").long("request-id") + .help("Retry an original UUIDv7 request; DATABASE must be its recorded Identity")) + .after_help("Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr.") + }; + command + .subcommand(base( + "status", + "Inspect container control state without opening its database", + )) + .subcommand(action("start", "Request container execution")) + .subcommand(action("stop", "Request container stop")) + .subcommand(action( + "restart", + "Request a new container instance and environment snapshot", + )) +} + +pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + validate_database(database)?; + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let supplied_request = args + .try_get_one::("request_id") + .ok() + .flatten() + .map(|id| Uuid::parse_str(id).context("request-id must be a UUIDv7")) + .transpose()?; + if supplied_request.is_some() { + ensure!( + database.parse::().is_ok(), + "retry with the original database Identity printed by the first command" + ); + } + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new( + server, + auth.to_header().context("container operations require a login")?, + )?; + if name == "status" { + let status = client.status(database).await?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&status)?); + } else { + print_status(&status); + } + return Ok(()); + } + let action = match name { + "start" => ContainerAction::Start, + "stop" => ContainerAction::Stop, + "restart" => ContainerAction::Restart, + _ => bail!("unsupported container operation"), + }; + // Names are resolved once before mutation. The replay target is immutable. + let identity = match database.parse::() { + Ok(identity) => identity, + Err(_) => client.status(database).await?.database_identity, + }; + let request_id = supplied_request.unwrap_or_else(|| Uuid::from_u128(uuid::Uuid::now_v7().as_u128())); + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + spacetimedb_lib::deployment::operation_expiry_ms(request_id, now) + .context("request ID is not within its seven-day retry window")?; + // Structured retry parameters preserve the selected origin without + // presenting unescaped server text as an executable shell command. + let retry = serde_json::json!({ + "command": format!("container {}", action.path()), + "database": identity.to_hex().to_string(), + "request_id": request_id.to_string(), + "server": client.server.as_str(), + }); + eprintln!("Container request retry parameters: {retry}"); + let receipt = client.operate(identity, request_id, action).await.with_context(|| { + format!("request did not return a verified receipt; retain these exact retry parameters: {retry}") + })?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "Accepted {} for {} at generation {}", + action.path(), + identity.to_hex(), + receipt.generation + ); + } + Ok(()) +} + +pub(super) struct ContainerClient { + pub(super) http: Client, + server: Url, + pub(super) authorization: HeaderValue, +} +impl ContainerClient { + pub(super) fn new(server: Url, mut authorization: HeaderValue) -> Result { + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "container operations require ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + pub(super) fn url(&self, database: &str, operation: &str) -> Result { + validate_database(database)?; + let mut url = self.server.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server origin"))? + .pop_if_empty() + .extend(["v1", "database", database, "container", operation]); + Ok(url) + } + pub(super) async fn status(&self, database: &str) -> Result { + let response = self + .http + .get(self.url(database, "status")?) + .header(AUTHORIZATION, self.authorization.clone()) + .send() + .await + .map_err(|_| anyhow::anyhow!("container status could not be reached on the selected server"))?; + let status: ContainerStatus = read_response(response, StatusCode::OK).await?; + if let Ok(identity) = database.parse::() { + ensure!( + status.database_identity == identity, + "status returned another database Identity" + ); + } + validate_status(&status)?; + Ok(status) + } + async fn operate( + &self, + identity: Identity, + request_id: Uuid, + action: ContainerAction, + ) -> Result { + let response = self + .http + .request(Method::POST, self.url(identity.to_hex().as_ref(), action.path())?) + .header(AUTHORIZATION, self.authorization.clone()) + .json(&ContainerOperationRequest { request_id }) + .send() + .await + .map_err(|_| anyhow::anyhow!("container operation response was not received"))?; + let receipt: ContainerOperationReceipt = read_response(response, StatusCode::ACCEPTED).await?; + ensure!( + receipt.database_identity == identity && receipt.request_id == request_id && receipt.action == action, + "container operation receipt does not match this request" + ); + Ok(receipt) + } +} +fn validate_database(database: &str) -> Result<()> { + ensure!( + !database.is_empty() + && database.len() <= 1024 + && !matches!(database, "." | "..") + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + Ok(()) +} +async fn read_response(mut response: reqwest::Response, expected: StatusCode) -> Result { + let status = response.status(); + ensure!( + response.content_length().is_none_or(|size| size <= MAX_RESPONSE as u64), + "container response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE.saturating_sub(bytes.len()), + "container response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != expected { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit this operation", + Some(ContainerErrorCode::NotFound) => "database was not found", + Some(ContainerErrorCode::InvalidRequest) => "container request is invalid or expired", + Some(ContainerErrorCode::Conflict) => "container request conflicts with current state", + Some(ContainerErrorCode::OutcomeUnknown) => + "container request outcome is unknown; retry the same request ID", + _ => "container service is unavailable on the selected server", + }, + status.as_u16() + ); + } + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container response")) +} +fn validate_status(status: &ContainerStatus) -> Result<()> { + if let Some(state) = &status.operational + && let Some(instance) = &state.current_instance + { + ensure!( + instance.generation == state.generation, + "status instance is not the current generation" + ); + if let Some(id) = &instance.applied_env_generation { + ensure!(Uuid::parse_str(id).is_ok(), "invalid applied environment generation"); + } + } + if let EndpointStatus::Available { endpoints } = &status.endpoints { + super::url::validate(&ContainerEndpoints { + database_identity: status.database_identity, + endpoints: endpoints.clone(), + })?; + } + Ok(()) +} +fn print_status(status: &ContainerStatus) { + println!("Database: {}", status.database_identity.to_hex()); + if !status.published { + println!("Container: none published"); + } + if let Some(configuration) = &status.configuration { + println!("Published image: {}", configuration.image_digest); + let limits = &configuration.resources; + println!( + "Published limits: {} millicores, {} memory bytes, {} scratch bytes, {} tasks", + limits.cpu_millicores, limits.memory_bytes, limits.scratch_bytes, limits.pids_max + ); + } + if let Some(state) = &status.operational { + println!("Desired: {:?} (generation {})", state.desired_state, state.generation); + println!("Condition: {:?}", state.condition); + if let Some(instance) = &state.current_instance { + println!("Observed: {:?}", instance.state); + if let Some(usage) = &instance.usage { + println!( + "Reported usage: sample {}{}", + usage.sample_sequence, + if usage.final_report { " (final totals)" } else { "" } + ); + let totals = &usage.cumulative; + println!( + "CPU: {} ns; memory: {} byte-seconds; scratch: {} byte-seconds; sent: {} bytes; received: {} bytes", + totals.cpu_nanoseconds, + totals.memory_byte_seconds, + totals.scratch_byte_seconds, + totals.transmitted_bytes, + totals.received_bytes + ); + if usage.measurement_interrupted { + println!("Measurement interrupted by host restart; totals include known usage only."); + } + } else { + println!("Usage: not reported for this generation"); + } + if let Some(environment) = &instance.applied_env_generation { + println!("Environment generation: {environment}"); + } + if let Some(code) = instance.exit_code { + println!("Exit code: {code}"); + } + if instance.oom_killed { + println!("Out of memory: yes"); + } + } else { + println!("Observed: no instance admitted for this generation"); + } + } + match &status.endpoints { + EndpointStatus::Available { endpoints } => { + for endpoint in endpoints { + println!("{}: {}", endpoint.name, endpoint.url); + } + } + EndpointStatus::Pending => println!("Endpoints: pending"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn target_path_encodes_one_database_and_rejects_dot_segments() { + let client = ContainerClient::new( + Url::parse("http://127.0.0.1:43123/").unwrap(), + HeaderValue::from_static("Bearer unusable"), + ) + .unwrap(); + assert_eq!( + client.url("name/child?x#y", "status").unwrap().path(), + "/v1/database/name%2Fchild%3Fx%23y/container/status" + ); + for invalid in ["", ".", "..", "name\n"] { + assert!(client.url(invalid, "status").is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/container/url.rs b/crates/cli/src/subcommands/container/url.rs new file mode 100644 index 00000000000..b66eee410aa --- /dev/null +++ b/crates/cli/src/subcommands/container/url.rs @@ -0,0 +1,172 @@ +//! Anonymous discovery prints one validated public address, never a constructed +//! hostname, authentication token or private runtime address. + +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgMatches, Command}; +use reqwest::{redirect::Policy, Client, StatusCode}; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, MAX_PORTS}, + Identity, +}; +use std::{collections::BTreeSet, time::Duration}; +use url::Url; + +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +pub(super) fn cli() -> Command { + Command::new("url") + .about("Print a container's published HTTPS URL") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg( + Arg::new("port") + .long("port") + .help("Declared port name; required when several ports are published"), + ) + .arg(crate::common_args::server()) + .after_help("Discovery does not start the container or wait for readiness. No login is required.") +} + +pub(super) async fn exec(config: &crate::Config, args: &ArgMatches) -> Result<()> { + let server = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + let port = args.get_one::("port").map(String::as_str); + let endpoints = fetch(&config.get_host_url(server)?, database).await?; + println!("{}", select(&endpoints, port)?); + Ok(()) +} + +async fn fetch(server: &str, database: &str) -> Result { + ensure!( + !database.is_empty() + && !matches!(database, "." | "..") + && database.len() <= 1024 + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + let mut url = Url::parse(server).context("invalid server URL")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && url.path() == "/", + "server must be an HTTP or HTTPS origin without credentials or a path" + ); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server URL"))? + .clear() + .extend(["v1", "database", database, "container", "endpoints"]); + // This endpoint is public. Do not load a login, resolve an Identity through + // another request, follow redirects, or forward saved credentials. + let client = Client::builder() + .no_proxy() + .redirect(Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(15)) + .build()?; + let mut response = client + .get(url) + .send() + .await + .map_err(|_| anyhow::anyhow!("could not reach container endpoint discovery on the selected server"))?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + bail!("database or container endpoint discovery was not found on the selected server"); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + bail!("container endpoints are not available yet; retry shortly"); + } + if status != StatusCode::OK { + bail!("container endpoint discovery failed (HTTP {})", status.as_u16()); + } + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_RESPONSE_BYTES as u64), + "container endpoint discovery response exceeds its size limit" + ); + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container endpoint discovery response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(body.len()), + "container endpoint discovery response exceeds its size limit" + ); + body.extend_from_slice(&chunk); + } + let endpoints: ContainerEndpoints = + serde_json::from_slice(&body).map_err(|_| anyhow::anyhow!("invalid container endpoint discovery response"))?; + if let Ok(identity) = database.parse::() { + ensure!( + identity == endpoints.database_identity, + "container endpoint discovery returned another database Identity" + ); + } + validate(&endpoints)?; + Ok(endpoints) +} + +pub(super) fn validate(endpoints: &ContainerEndpoints) -> Result<()> { + ensure!( + endpoints.endpoints.len() <= MAX_PORTS, + "too many container endpoints in discovery response" + ); + let mut names = BTreeSet::new(); + for endpoint in &endpoints.endpoints { + let name = endpoint.name.as_bytes(); + ensure!( + (1..=32).contains(&name.len()) + && name[0].is_ascii_lowercase() + && name + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') + && names.insert(&endpoint.name), + "invalid or duplicate port name in discovery response" + ); + let address = Url::parse(&endpoint.url).context("invalid container endpoint URL")?; + ensure!( + endpoint.url.len() <= 2048 + && endpoint.url.is_ascii() + && !endpoint.url.bytes().any(|byte| byte.is_ascii_control()) + && address.as_str() == endpoint.url + && address.scheme() == "https" + && matches!(address.host(), Some(url::Host::Domain(_))) + && address.username().is_empty() + && address.password().is_none() + && address.port().is_none() + && address.query().is_none() + && address.fragment().is_none() + && address.path() == "/", + "discovery returned an invalid public HTTPS endpoint" + ); + } + Ok(()) +} + +fn select<'a>(endpoints: &'a ContainerEndpoints, port: Option<&str>) -> Result<&'a str> { + match (port, endpoints.endpoints.as_slice()) { + (_, []) => bail!("this database has no declared public HTTP ports"), + (None, [endpoint]) => Ok(&endpoint.url), + (Some(port), entries) => entries + .iter() + .find(|endpoint| endpoint.name == port) + .map(|endpoint| endpoint.url.as_str()) + .context("the requested port name is not published by this database"), + (None, entries) => bail!( + "several ports are published; select one with --port: {}", + entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>() + .join(", ") + ), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/url/tests.rs b/crates/cli/src/subcommands/container/url/tests.rs new file mode 100644 index 00000000000..37a544d8b75 --- /dev/null +++ b/crates/cli/src/subcommands/container/url/tests.rs @@ -0,0 +1,198 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::Path, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use spacetimedb_lib::container::{endpoints::ContainerEndpoint, PortProtocol}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn response() -> ContainerEndpoints { + ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + } +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} + +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn discovery_is_anonymous_and_encodes_the_entire_database_path_segment() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path, headers: HeaderMap| { + let count = count.clone(); + async move { + assert_eq!(database, "project/child?query=value#fragment"); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + count.fetch_add(1, Ordering::SeqCst); + Json(response()) + } + }), + )) + .await?; + for invalid in [".", ".."] { + assert!(fetch(&server.origin, invalid).await.is_err()); + } + assert_eq!(requests.load(Ordering::SeqCst), 0); + let discovered = fetch(&server.origin, "project/child?query=value#fragment").await?; + assert_eq!(select(&discovered, None)?, response().endpoints[0].url); + assert_eq!(requests.load(Ordering::SeqCst), 1); + server.shutdown().await +} + +#[test] +fn port_selection_never_picks_an_arbitrary_endpoint() -> Result<()> { + let mut endpoints = response(); + assert_eq!(select(&endpoints, Some("http"))?, endpoints.endpoints[0].url); + assert!(select(&endpoints, Some("absent")).is_err()); + let mut second = endpoints.endpoints[0].clone(); + second.name = "metrics".into(); + second.url = "https://bbbbbbbbbbbbbbbbbbbbbbbbaa.container.example.net/".into(); + endpoints.endpoints.push(second); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("--port: http, metrics")); + assert_eq!(select(&endpoints, Some("metrics"))?, endpoints.endpoints[1].url); + endpoints.endpoints.clear(); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("no declared public HTTP ports")); + Ok(()) +} + +#[test] +fn endpoint_output_rejects_terminal_controls_credentials_and_ambiguous_names() { + let mut endpoints = response(); + for address in [ + "javascript:alert(1)", + "http://application.example/", + "https://user:secret@application.example/", + "https://127.0.0.1/", + "https://application.example/path", + "https://application.example/?secret=value", + "https://application.example/#fragment", + "https://application.example:8443/", + "https://application.example/\n", + "\u{1b}[2Jhttps://application.example/", + ] { + endpoints.endpoints[0].url = address.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + for name in ["", "HTTP", "1http", "http\n", "http/metrics"] { + endpoints.endpoints[0].name = name.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + endpoints.endpoints.push(endpoints.endpoints[0].clone()); + assert!(validate(&endpoints).is_err()); +} + +#[tokio::test] +async fn discovery_bounds_streams_rejects_redirects_and_preserves_pending_errors() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path| { + let count = count.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + match database.as_str() { + "pending" => (StatusCode::SERVICE_UNAVAILABLE, "private diagnostic").into_response(), + "missing" => StatusCode::NOT_FOUND.into_response(), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_RESPONSE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + _ => Json(response()).into_response(), + } + } + }), + )) + .await?; + for (database, expected) in [ + ("pending", "not available yet"), + ("missing", "not found"), + ("redirect", "HTTP 302"), + ("invalid", "invalid container endpoint discovery response"), + ("oversize", "size limit"), + ] { + let error = format!("{:#}", fetch(&server.origin, database).await.unwrap_err()); + assert!(error.contains(expected), "{error}"); + assert!(!error.contains("private diagnostic")); + } + let other_identity = Identity::from_u256(1_u64.into()).to_string(); + assert!(fetch(&server.origin, &other_identity) + .await + .unwrap_err() + .to_string() + .contains("another database Identity")); + assert_eq!(requests.load(Ordering::SeqCst), 6); + server.shutdown().await +} + +#[test] +fn url_command_requires_database_and_parses_explicit_server_and_port() { + assert!(cli().try_get_matches_from(["url"]).is_err()); + let args = cli() + .try_get_matches_from(["url", "demo", "--server", "http://127.0.0.1:3000", "--port", "http"]) + .unwrap(); + assert_eq!(args.get_one::("database").unwrap(), "demo"); + assert_eq!(args.get_one::("server").unwrap(), "http://127.0.0.1:3000"); + assert_eq!(args.get_one::("port").unwrap(), "http"); +} diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index 95cc579182f..1e91ac43c22 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -958,6 +958,10 @@ fn determine_publish_configs<'a>( } if !publish_configs.is_empty() { + anyhow::ensure!( + publish_configs.iter().all(|target| target.container().is_none()), + "spacetime dev does not yet run container targets; use spacetime publish for managed publication" + ); return Ok(publish_configs); } diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index 9c659a5880f..8dccf5cbdf4 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,5 +1,6 @@ pub mod build; pub mod call; +pub mod container; pub mod db_arg_resolution; pub mod delete; pub mod describe; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 5162dbeca2c..3816189c951 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -24,6 +24,8 @@ use crate::util::{add_auth_header_opt, get_auth_header, strip_verbatim_prefix, A use crate::util::{decode_identity, y_or_n}; use crate::{build, common_args}; +mod managed; + /// Individual prompts that `--yes` can suppress. `All` is a shorthand for every category below. #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] #[clap(rename_all = "kebab-case")] @@ -88,7 +90,7 @@ fn yes_flags_from_args(args: &ArgMatches) -> YesFlags { /// Build the CommandSchema for publish command pub fn build_publish_schema(command: &clap::Command) -> Result { - CommandSchemaBuilder::new() + managed::exclude_args(CommandSchemaBuilder::new()) .key(Key::new("database").from_clap("name|identity").required()) .key(Key::new("server")) .key(Key::new("env").config_only()) @@ -178,7 +180,7 @@ pub fn get_filtered_publish_configs<'a>( let configs: Vec = filtered_targets .into_iter() .map(|target| { - let config = CommandConfig::new(schema, target.fields, args)?; + let config = CommandConfig::new(schema, target.fields, args)?.with_container(target.container); config.validate()?; Ok(config) }) @@ -196,7 +198,7 @@ pub fn get_filtered_publish_configs<'a>( } pub fn cli() -> clap::Command { - clap::Command::new("publish") + managed::add_args(clap::Command::new("publish") .about("Create and update a SpacetimeDB database") .arg(common_args::clear_database()) .arg( @@ -325,7 +327,7 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), ) .arg(common_args::dotnet_version()) .after_help("Run `spacetime help publish` for more detailed information.") - .after_long_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.") + .after_long_help("Every publish replaces the complete declared environment. Put an env map in spacetime.json; declared shell variables override config values (including empty strings). The CLI displays supplied keys and sources, never values. Optional values omitted from every input are removed. --env selects config file layers. Run `spacetime help publish` for more detailed information.")) } fn publication_body( @@ -407,6 +409,9 @@ pub async fn exec_with_options( quiet_config: bool, pre_loaded_config: Option<&LoadedConfig>, ) -> Result<(), anyhow::Error> { + if args.get_one::("resume_publication").is_some() { + return managed::resume(&mut config, args, yes_flags_from_args(args)).await; + } // Build schema let cmd = cli(); let schema = build_publish_schema(&cmd)?; @@ -494,6 +499,15 @@ pub async fn exec_from_entry( execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await } +fn dotnet_version_from_config(command_config: &CommandConfig<'_>) -> anyhow::Result> { + if command_config.is_from_cli("dotnet_version") { + Ok(command_config.get_one::("dotnet_version")?) + } else { + let dotnet_version = command_config.get_one::("dotnet_version")?; + parse_optional_dotnet_version(dotnet_version.as_deref()) + } +} + async fn execute_publish_configs<'a>( config: &mut Config, publish_configs: Vec>, @@ -522,20 +536,7 @@ async fn execute_publish_configs<'a>( }) }; - if using_config { - if let Some(path_to_project) = path_to_project.as_ref() { - println!( - "Publishing module {} to database '{}'", - strip_verbatim_prefix(path_to_project).display(), - name_or_identity.unwrap() - ); - } else { - println!( - "Publishing precompiled module to database '{}'", - name_or_identity.unwrap() - ); - } - } + managed::validate_target_options(&command_config)?; let database_host = config.get_host_url(server)?; let build_options = command_config .get_one::("build_options")? @@ -547,12 +548,7 @@ async fn execute_publish_configs<'a>( let org_opt = command_config.get_one::("organization")?; let org = org_opt.as_deref(); let native_aot = command_config.get_one::("native_aot")?.unwrap_or(false); - let dotnet_version = if command_config.is_from_cli("dotnet_version") { - command_config.get_one::("dotnet_version")? - } else { - let dotnet_version = command_config.get_one::("dotnet_version")?; - parse_optional_dotnet_version(dotnet_version.as_deref())? - }; + let dotnet_version = dotnet_version_from_config(&command_config)?; // If the user didn't specify an identity and we didn't specify an anonymous identity, then // we want to use the default identity @@ -562,6 +558,36 @@ async fn execute_publish_configs<'a>( let (name_or_identity, parent) = validate_name_and_parent(name_or_identity, parent)?; + if managed::try_execute( + &command_config, + config_dir, + &database_host, + &auth_header, + name_or_identity, + parent, + clear_database, + yes, + ) + .await? + { + continue; + } + + if using_config { + if let Some(path_to_project) = path_to_project.as_ref() { + println!( + "Publishing module {} to database '{}'", + strip_verbatim_prefix(path_to_project).display(), + name_or_identity.unwrap() + ); + } else { + println!( + "Publishing precompiled module to database '{}'", + name_or_identity.unwrap() + ); + } + } + if let Some(path_to_project) = path_to_project.as_ref() && !path_to_project.exists() { @@ -1329,6 +1355,40 @@ mod tests { assert_eq!(matches.get_one::("dotnet_version").copied(), Some(10)); } + #[test] + fn managed_publish_preserves_dotnet_selection_and_selective_yes() { + let command = cli(); + let schema = build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "--managed", + "--dotnet-version", + "10", + "--yes=remote,skip-login", + "--server", + "http://127.0.0.1:1", + "test-db", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + target.validate().unwrap(); + assert!(args.get_flag("managed")); + assert_eq!(dotnet_version_from_config(&target).unwrap(), Some(10)); + let yes = yes_flags_from_args(&args); + assert!(yes.publish_to_remote && yes.skip_login); + assert!(!yes.migrate_major_version && !yes.break_clients && !yes.delete_data); + + let args = cli().get_matches_from(["publish", "--managed", "test-db"]); + let target = CommandConfig::new( + &schema, + HashMap::from([("dotnet_version".into(), serde_json::json!("10"))]), + &args, + ) + .unwrap(); + assert_eq!(dotnet_version_from_config(&target).unwrap(), Some(10)); + } + #[test] fn test_publish_cli_rejects_unsupported_dotnet_version() { assert!(cli() diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs new file mode 100644 index 00000000000..983c083a466 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -0,0 +1,1100 @@ +//! Managed publication frontend. The resume journal owns every byte needed to +//! retry; project configuration and mutable image tags are only read initially. +use super::{confirm_major_version_upgrade, environment, YesFlags}; +use crate::{ + common_args::ClearMode, + config::Config, + container::{ + self, + oci::ArtifactKind, + process::LocalRunner, + publish::{ + self, + client::{ObjectRef, PublisherClient, UploadKind}, + journal::{Journal, Record, UploadRecord}, + Outcome, + }, + BuildSecret, BuildTools, + }, + spacetime_config::{CommandConfig, CommandSchemaBuilder}, + util::{get_auth_header, y_or_n, AuthHeader}, +}; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_client_api_messages::name::{is_identity, DomainName, PrePublishResult}; +use spacetimedb_lib::{ + container::{ContainerAction, ImagePlatform}, + deployment::{self, api::*, manifest::*, ModuleAction, PublishEnvelope, UserModule, UserModuleKind}, + Identity, Uuid, +}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; +use tokio_util::sync::CancellationToken; + +const ARGUMENTS: &[&str] = &[ + "managed", + "container_platform", + "artifact_endpoint", + "remove_container", + "remove_module", + "publication_state_dir", + "resume_publication", + "publication_wait", + "buildkit_host", + "buildctl", + "railpack", + "skopeo", + "registry_auth_file", + "build_secret", +]; +pub(super) fn exclude_args(mut schema: CommandSchemaBuilder) -> CommandSchemaBuilder { + for arg in ARGUMENTS { + schema = schema.exclude(*arg); + } + schema +} +pub(super) fn add_args(mut command: Command) -> Command { + command = command + .arg( + Arg::new("managed") + .long("managed") + .action(ArgAction::SetTrue) + .help("Use managed deployment publication, including for a module-only database"), + ) + .arg( + Arg::new("container_platform") + .long("container-platform") + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Required platform when publishing a container declaration"), + ) + .arg( + Arg::new("artifact_endpoint") + .long("artifact-endpoint") + .help("Explicitly authorize this exact artifact URL to receive the publisher credential"), + ) + .arg( + Arg::new("remove_container") + .long("remove-container") + .action(ArgAction::SetTrue) + .help("Explicitly remove the container while preserving the module unless separately changed"), + ) + .arg( + Arg::new("remove_module") + .long("remove-module") + .action(ArgAction::SetTrue) + .conflicts_with_all(["module_path", "wasm_file", "js_file"]) + .help("Replace the module with the versioned empty module after migration preflight"), + ) + .arg( + Arg::new("publication_state_dir") + .long("publication-state-dir") + .value_parser(clap::value_parser!(PathBuf)) + .help("Private local directory retaining complete managed publication inputs and progress") + .long_help("Private local directory retaining complete managed publication inputs and progress. The protected submission file includes resolved environment values. Keep this directory private; do not commit it or share it. Resume reuses these exact values without rereading project configuration or shell variables."), + ) + .arg( + Arg::new("resume_publication") + .long("resume-publication") + .value_parser(clap::value_parser!(PathBuf)) + .conflicts_with_all([ + "managed", + "remove_container", + "remove_module", + "module_path", + "wasm_file", + "js_file", + "container_platform", + "name|identity", + "parent", + "organization", + "clear-database", + ]) + .help("Resume this operation directory without rebuilding or reading spacetime.json"), + ) + .arg( + Arg::new("publication_wait") + .long("publication-wait") + .default_value("60") + .value_parser(clap::value_parser!(u64).range(0..=3600)) + .help("Seconds to wait for managed activation; pending operations retain their resume directory"), + ) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket for container source builds"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; otherwise image preparation is anonymous"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Build secret file, separate from runtime env_keys"), + ); + for (tool, help) in [ + ( + "buildctl", + "Path to the buildctl executable for Dockerfile or Railpack builds", + ), + ( + "railpack", + "Path to the Railpack executable for explicitly selected Railpack builds", + ), + ( + "skopeo", + "Path to the skopeo executable for copying prebuilt OCI images", + ), + ] { + command = command.arg( + Arg::new(tool) + .help(help) + .long(tool) + .default_value(tool) + .value_parser(clap::value_parser!(PathBuf)), + ); + } + command +} +fn approved(args: &ArgMatches) -> Option<&str> { + args.get_one::("artifact_endpoint").map(String::as_str) +} +fn wait(args: &ArgMatches) -> Duration { + Duration::from_secs(*args.get_one::("publication_wait").unwrap_or(&60)) +} +pub(super) fn validate_target_options(target: &CommandConfig<'_>) -> Result<()> { + let args = target.matches(); + if target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module") + { + ensure!( + !target.get_one::("anon_identity")?.unwrap_or(false), + "managed publication requires an authenticated publisher" + ); + } + Ok(()) +} +fn authenticated(server: &str, auth: &AuthHeader) -> Result { + PublisherClient::new( + server, + auth.to_header() + .context("managed publication requires an authenticated publisher; anonymous publication is unsupported")?, + ) +} +pub(super) async fn resume(config: &mut Config, args: &ArgMatches, yes: YesFlags) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = config.get_host_url(selection)?; + let mut journal = Journal::open(args.get_one::("resume_publication").unwrap())?; + ensure!( + container::publish::client::endpoint(&server)?.as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let auth = get_auth_header(config, false, selection, !yes.skip_login).await?; + let client = authenticated(&server, &auth)?; + execute(&client, &mut journal, args).await +} + +/// False is only returned for an ordinary, un-managed module publication. +/// Once managed intent or a managed revision is known, errors cannot fall back. +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_execute( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + server: &str, + auth: &AuthHeader, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + if !explicit && name.is_none() { + return Ok(false); + } + let client = match authenticated(server, auth) { + Ok(client) => client, + Err(error) if explicit => return Err(error), + Err(_) => return Ok(false), + }; + try_execute_with_client(target, config_dir, client, name, parent, clear, yes).await +} + +async fn try_execute_with_client( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + client: PublisherClient, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + let caps = client.capabilities().await?; + let Some(caps) = caps else { + ensure!( + !explicit, + "this server does not support managed publication; no module-only fallback was attempted" + ); + return Ok(false); + }; + let prior = if let Some(name) = name { + client.deployment(name).await? + } else { + None + }; + if !explicit && prior.as_ref().and_then(|p| p.revision).is_none() { + return Ok(false); + } + ensure!( + caps.enabled && caps.version == deployment::PUBLISH_PROTOCOL_VERSION, + "managed publication is disabled or incompatible on this server" + ); + ensure!( + clear == ClearMode::Never, + "managed publication does not support --delete-data; resolve migrations without replacing database storage" + ); + ensure!( + !(target.container().is_some() && args.get_flag("remove_container")), + "a container declaration and --remove-container cannot both select this target" + ); + if prior.is_none() { + ensure!( + !name.is_some_and(is_identity), + "a new database Identity must be generated by the reservation service; select a name or omit DATABASE" + ); + } + let artifact = client.artifact_endpoint( + caps.artifact_endpoint + .as_deref() + .context("server did not advertise an artifact endpoint")?, + approved(args), + )?; + let permission = client.permission().await?; + if target.container().is_some() || prior.is_none() { + ensure!( + permission.can_publish, + "the current publisher does not have container publication permission" + ); + } + if !container::publish::client::is_loopback(client.server()) { + ensure!( + y_or_n( + yes.publish_to_remote, + "Publish this managed deployment to the selected remote server?" + )?, + "publication cancelled" + ); + } + let cwd = std::env::current_dir()?; + let base = args + .get_one::("publication_state_dir") + .cloned() + .unwrap_or_else(|| config_dir.unwrap_or(&cwd).join(".spacetime/publications")); + let _base_parents = Journal::prepare_base(&base)?; + let cancel = CancellationToken::new(); + let prepare = prepare_request( + &client, + target, + config_dir.unwrap_or(&cwd), + &base, + prior.as_ref(), + name, + parent, + permission.identity, + artifact.as_str(), + yes, + cancel.clone(), + ); + tokio::pin!(prepare); + let mut journal = tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; cancel.cancel(); let _ = prepare.await; + bail!("managed preparation cancelled before admission"); + } + }; + execute(&client, &mut journal, args).await?; + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_request( + client: &PublisherClient, + target: &CommandConfig<'_>, + config_dir: &Path, + base: &Path, + prior: Option<&DeploymentStatus>, + name: Option<&str>, + parent: Option<&str>, + publisher: Identity, + artifact_endpoint: &str, + yes: YesFlags, + cancel: CancellationToken, +) -> Result { + let args = target.matches(); + let image = if let Some(declaration) = target.container() { + let (os, architecture) = args + .get_one::("container_platform") + .context("publishing a container requires --container-platform linux/amd64 or linux/arm64")? + .split_once('/') + .unwrap(); + Some( + container::prepare_container( + declaration, + config_dir, + ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools(args)?, + base, + &LocalRunner, + cancel.clone(), + ) + .await?, + ) + } else { + None + }; + let has_module = ["module_path", "wasm_file", "js_file"] + .iter() + .any(|key| target.is_from_cli(key) || target.get_config_value(key).is_some()); + ensure!( + !(has_module && args.get_flag("remove_module")), + "module configuration and --remove-module cannot both select this target" + ); + let module = if !args.get_flag("remove_module") + && (has_module || (target.container().is_none() && !args.get_flag("remove_container"))) + { + Some(load_module(target, config_dir).await?) + } else { + None + }; + let declared_environment = target + .container() + .map(|config| config.environment_schema()) + .transpose()? + .flatten(); + ensure!( + declared_environment.is_none() || module.is_none(), + "container env_schema cannot override a user module's environment declarations" + ); + ensure!( + declared_environment.is_none() + || args.get_flag("remove_module") + || !prior + .is_some_and(|prior| matches!(prior.deployment.current().module, deployment::ModuleComponent::User(_))), + "replacing a user module with container env_schema requires --remove-module" + ); + let builtin = + if module.is_none() && (args.get_flag("remove_module") || declared_environment.is_some() || prior.is_none()) { + Some(deployment::system_empty::generate( + &declared_environment.unwrap_or_default(), + )?) + } else { + None + }; + let module_action = if let Some(builtin) = &builtin { + ModuleAction::Remove(builtin.descriptor) + } else if let Some((kind, bytes)) = &module { + ModuleAction::Set(UserModule { + kind: *kind, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }) + } else { + ModuleAction::Keep + }; + let module = builtin + .map(|builtin| (UserModuleKind::Wasm, builtin.bytes.into_vec())) + .or(module); + let container_action = if let Some(image) = &image { + ContainerAction::Set(image.metadata.container.clone()) + } else if args.get_flag("remove_container") { + ContainerAction::Remove + } else { + ContainerAction::Keep + }; + let envelope = PublishEnvelope { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: prior.and_then(|p| p.revision), + expected_last_operation: prior.and_then(|p| p.last_operation), + module_action, + container_action, + }; + let deployment = envelope.resolve(prior.map(|p| &p.deployment), &Default::default())?; + let module_artifact = if let Some((_, bytes)) = &module { + ModuleArtifact { + digest: spacetimedb_oci::sha256(bytes), + size_bytes: bytes.len() as u64, + } + } else if let Some(prior) = prior { + let artifact = prior.module_artifact; + ModuleArtifact { + digest: artifact.digest, + size_bytes: artifact.size_bytes, + } + } else { + bail!("publication has no selected module artifact") + }; + // Resolve a complete replacement without consulting the stored values. + // For Keep, authenticated schema metadata is tied to the observed program; + // a changed committed operation is rejected by the later pair CAS. + let schema = if let Some((kind, bytes)) = &module { + selected_environment(*kind, bytes).await? + } else { + client + .selected_environment(prior.context("publication has no selected module")?) + .await? + }; + let environment = environment::resolve(&schema, target.get_config_value("env"), |key| std::env::var_os(key))?; + print!("{}", environment.display()); + let migration_policy = if let Some(prior) = prior { + if let Some((kind, bytes)) = &module { + migration(client, prior.database_identity, *kind, bytes, target, yes).await? + } else { + PreparedMigrationPolicy::Compatible + } + } else { + PreparedMigrationPolicy::Compatible + }; + let creation = if prior.is_none() { + let parent = if let Some(parent) = parent { + Some( + client + .deployment(parent) + .await? + .context("parent database does not exist or is not accessible")? + .database_identity, + ) + } else { + None + }; + let organization = target + .get_one::("organization")? + .map(|value| { + value + .parse::() + .context("managed publication currently requires --organization IDENTITY, not an organization name") + }) + .transpose()?; + Some(CreationOptions { + parent, + organization, + num_replicas: target.get_one::("num_replicas")?.map(u32::from), + enforce_anti_affinity: true, + }) + } else { + None + }; + let request = PublishRequest { + environment: environment.values, + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope, + deployment, + module_artifact, + migration_policy, + }), + creation, + image_source: image.as_ref().map(|image| ArtifactReference { + digest: image.metadata.manifest.digest, + size_bytes: image.metadata.manifest.size, + }), + }; + request.manifest.validate(&Default::default())?; + let request_json = serde_json::to_string(&request)?; + let mut uploads = image + .as_ref() + .map(|image| { + image + .metadata + .objects + .iter() + .map(|object| UploadRecord { + kind: match object.kind { + ArtifactKind::Manifest => UploadKind::Manifest, + ArtifactKind::Config => UploadKind::Config, + ArtifactKind::Layer => UploadKind::Layer, + }, + object: ObjectRef { + digest: object.descriptor.digest, + size: object.descriptor.size, + }, + session: None, + }) + .collect::>() + }) + .unwrap_or_default(); + if module.is_some() { + uploads.push(UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }); + } + let requested_name = if prior.is_none() { + name.map(|name| name.parse::().map(|name| name.to_string())) + .transpose()? + } else { + None + }; + let record = Record { + version: 2, + server: client.server().to_string(), + artifact_endpoint: artifact_endpoint.into(), + publisher, + database: prior.map(|p| p.database_identity), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name, + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads, + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + }; + Journal::create(base, record, image, module.as_ref().map(|(_, bytes)| bytes.as_slice())) +} +async fn selected_environment( + kind: UserModuleKind, + bytes: &[u8], +) -> Result { + // Canonical platform modules can be inspected without executing a helper. + // The verifier regenerates every byte; this is equally valid if a user + // explicitly selected those exact canonical Wasm bytes with --bin-path. + if kind == UserModuleKind::Wasm { + let descriptor = deployment::system_empty::SystemEmptyModule { + version: deployment::system_empty::VERSION, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }; + if let Ok(schema) = deployment::system_empty::verify(&descriptor, bytes) { + return Ok(schema); + } + } + let host_type = match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }; + Ok(environment::inspect(bytes, host_type).await?.environment().clone()) +} + +fn tools(args: &ArgMatches) -> Result { + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for value in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = value.split_once('=').context("build secret must be NAME=FILE")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + Ok(tools) +} +async fn load_module(target: &CommandConfig<'_>, config_dir: &Path) -> Result<(UserModuleKind, Vec)> { + let native_aot = target.get_one::("native_aot")?.unwrap_or(false); + ensure!( + !native_aot, + "managed NativeAOT builds are not yet supported; build separately and pass --bin-path" + ); + let (path, kind) = if let Some(path) = target.get_resolved_path("wasm_file", Some(config_dir))? { + (path, "Wasm") + } else if let Some(path) = target.get_resolved_path("js_file", Some(config_dir))? { + (path, "Js") + } else { + let path = target + .get_resolved_path("module_path", Some(config_dir))? + .unwrap_or_else(|| super::default_publish_module_path(config_dir)); + crate::build::exec_with_argstring( + &path, + &target.get_one::("build_options")?.unwrap_or_default(), + native_aot, + super::dotnet_version_from_config(target)?, + ) + .await? + }; + let kind = match kind { + "Wasm" => UserModuleKind::Wasm, + "Js" => UserModuleKind::Js, + _ => bail!("unsupported managed module kind"), + }; + let metadata = tokio::fs::metadata(&path).await?; + ensure!( + metadata.is_file() && metadata.len() > 0 && metadata.len() <= MAX_MODULE_ARTIFACT_BYTES, + "module must be a regular file of at most 32 MiB" + ); + use tokio::io::AsyncReadExt as _; + let mut bytes = Vec::new(); + tokio::fs::File::open(&path) + .await? + .take(MAX_MODULE_ARTIFACT_BYTES + 1) + .read_to_end(&mut bytes) + .await?; + ensure!( + !bytes.is_empty() && bytes.len() as u64 <= MAX_MODULE_ARTIFACT_BYTES, + "module exceeds 32 MiB" + ); + Ok((kind, bytes)) +} +async fn migration( + client: &PublisherClient, + database: Identity, + kind: UserModuleKind, + bytes: &[u8], + target: &CommandConfig<'_>, + yes: YesFlags, +) -> Result { + let pre = client + .preflight( + database, + match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }, + bytes, + ) + .await?; + match pre { + PrePublishResult::ManualMigrate(_) => { + bail!("managed publication requires manual migration; no storage was cleared") + } + PrePublishResult::AutoMigrate(auto) => { + if auto.major_version_upgrade { + confirm_major_version_upgrade(yes.migrate_major_version)?; + } + println!("{}", auto.migrate_plan); + if auto.break_clients { + ensure!( + y_or_n( + yes.break_clients || target.get_one::("break_clients")?.unwrap_or(false), + "These changes will BREAK existing clients. Proceed?" + )?, + "publication cancelled" + ); + } + Ok(PreparedMigrationPolicy::BreakClients(auto.token)) + } + } +} +async fn execute(client: &PublisherClient, journal: &mut Journal, args: &ArgMatches) -> Result<()> { + println!( + "Publication {}. Resume directory: {}", + journal.operation()?, + journal.directory().display() + ); + let cancel = CancellationToken::new(); + let operation = publish::run(client, journal, approved(args), wait(args), cancel.clone()); + tokio::pin!(operation); + let outcome = tokio::select! { + result = &mut operation => result?, + signal = tokio::signal::ctrl_c() => { signal?; cancel.cancel(); bail!("publication interrupted; resume the same directory to determine its outcome"); } + }; + match outcome { + Outcome::Complete(status) => println!( + "Deployment {} is active on {}", + status.proposed_revision, status.database_identity + ), + Outcome::Pending(status) => println!( + "Publication {} is {:?} on {}; resume to observe activation", + status.operation_id, status.phase, status.database_identity + ), + Outcome::Aborted(status) => bail!( + "publication {} was aborted before commit on {}; resume state retained", + status.operation_id, + status.database_identity + ), + Outcome::NamingUnconfirmed(status) => bail!( + "deployment is active on {}, but naming was not confirmed; publication must not be repeated", + status.database_identity + ), + } + Ok(()) +} + +#[cfg(test)] +mod environment_tests; + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::publish::tests::{database, Fixture}; + use crate::spacetime_config::SpacetimeConfig; + use serde_json::json; + use spacetimedb_paths::FromPathUnchecked as _; + use std::collections::HashMap; + + #[tokio::test] + async fn container_only_frontend_uploads_verified_closure_with_server_reserved_identity() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let layout = temporary.path().join("layout"); + let selected = crate::container::tests::fixture(&layout); + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:layout"})); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let state = temporary.path().join("new/state"); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for parent in [temporary.path().join("new"), state.clone()] { + assert_eq!(std::fs::metadata(parent).unwrap().permissions().mode() & 0o777, 0o700); + } + } + { + let snapshot = fixture.state.lock().unwrap(); + assert_eq!(snapshot.reservations.len(), 1); + assert_eq!(snapshot.submits.len(), 1); + assert_eq!(snapshot.begin_count, 4); + let request: PublishRequest = serde_json::from_slice(&snapshot.submits[0]).unwrap(); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Remove(_) + )); + assert_eq!( + request.manifest.current().deployment.current().module, + deployment::ModuleComponent::SystemEmpty(deployment::system_empty::empty().descriptor) + ); + assert_eq!( + request.manifest.current().module_artifact, + crate::container::publish::tests::empty_module_artifact() + ); + assert_eq!(request.image_source.unwrap().digest, selected.digest); + let dir = state.join(request.manifest.current().envelope.operation_id.to_string()); + let journal = Journal::open(&dir).unwrap(); + assert_eq!(journal.record.database, Some(database())); + let bytes = std::fs::read_to_string(dir.join("publication.json")).unwrap(); + assert!(!bytes.contains("private-image-value")); + } + fixture.close().await; + } + #[cfg(unix)] + #[tokio::test] + async fn writable_publication_parent_is_rejected_before_image_preparation_or_submission() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let state = temporary.path().join("state"); + std::fs::create_dir(&state).unwrap(); + std::fs::set_permissions(&state, std::fs::Permissions::from_mode(0o777)).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + ]) + .unwrap(); + // If preparation runs, this missing local source would fail first. + // No real image helper, registry, or saved configuration is used. + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:missing-fixture"})); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + let error = try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all(), + ) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("untrusted writable ancestor")); + assert_eq!(std::fs::metadata(&state).unwrap().permissions().mode() & 0o777, 0o777); + assert_eq!(std::fs::read_dir(&state).unwrap().count(), 0); + { + let state = fixture.state.lock().unwrap(); + assert!(state.reservations.is_empty()); + assert!(state.submits.is_empty()); + assert_eq!(state.begin_count, 0); + } + fixture.close().await; + } + + #[tokio::test] + async fn existing_managed_module_update_keeps_container_and_preflights_without_pro() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let wasm = temporary.path().join("module.wasm"); + std::fs::write(&wasm, deployment::system_empty::empty().bytes.as_ref()).unwrap(); + let prior_request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + revision: Some(prior_request.manifest.current().deployment.revision().unwrap()), + deployment: prior_request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: crate::container::publish::tests::empty_module_artifact().digest, + size_bytes: crate::container::publish::tests::empty_module_artifact().size_bytes, + }, + }; + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.prior = Some(prior.clone()); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--bin-path", + wasm.to_str().unwrap(), + "--publication-state-dir", + temporary.path().join("state").to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.reservations.len(), 0); + let request: PublishRequest = serde_json::from_slice(&state.submits[0]).unwrap(); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.container_action, + ContainerAction::Keep + )); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Set(_) + )); + } + fixture.close().await; + } + #[tokio::test] + async fn remove_module_requires_authorized_preflight_before_any_upload_or_submission() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let request = fixture.record(false, false).request().unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_preflight = true; + state.prior = Some(DeploymentStatus { + database_identity: database(), + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: crate::container::publish::tests::empty_module_artifact().digest, + size_bytes: crate::container::publish::tests::empty_module_artifact().size_bytes, + }, + }); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--remove-module", + "--publication-state-dir", + temporary.path().to_str().unwrap(), + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .is_err()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.begin_count, 0); + assert!(state.submits.is_empty()); + } + fixture.close().await; + } + #[tokio::test] + async fn ordinary_module_target_returns_to_legacy_only_when_no_managed_revision_exists() { + let fixture = Fixture::new().await; + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "fixture-name", "--server", fixture.endpoint.as_str()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(!try_execute_with_client( + &target, + None, + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } + #[tokio::test] + async fn resume_entrypoint_ignores_changed_project_and_reuses_original_bytes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut record = fixture.record(false, false); + let mut request = record.request().unwrap(); + request + .environment + .insert("ORIGINAL_ENV".into(), "original-secret-value".into()); + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::system_empty::empty().bytes.as_ref()), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let resume = journal.directory().to_owned(); + drop(journal); + let cli_config = temporary.path().join("isolated-cli.toml"); + std::fs::write(&cli_config, "spacetimedb_token = 'isolated-fixture-credential'\n").unwrap(); + let config = Config::load(spacetimedb_paths::cli::CliTomlPath::from_path_unchecked(cli_config)).unwrap(); + let changed_project = crate::spacetime_config::LoadedConfig { + config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias", "env":{"ORIGINAL_ENV":"changed-secret-value"}})).unwrap(), + config_dir: temporary.path().into(), loaded_files: vec![], has_dev_file: false, + }; + let args = super::super::cli() + .try_get_matches_from([ + "publish", + "--resume-publication", + resume.to_str().unwrap(), + "--server", + fixture.endpoint.as_str(), + "--publication-wait", + "0", + ]) + .unwrap(); + super::super::exec_with_options(config, &args, true, Some(&changed_project)) + .await + .unwrap(); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; + } + #[test] + fn nested_targets_keep_only_their_own_container_and_resume_rejects_new_inputs() { + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"parent", "container":{"image":{"oci_ref":"oci:layout"},"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}}, "children":[{"database":"child"}]})).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.clone().try_get_matches_from(["publish"]).unwrap(); + let targets = super::super::get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert_eq!(targets.len(), 2); + assert!(targets[0].container().is_some()); + assert!(targets[1].container().is_none()); + for flags in [ + ["--resume-publication", "operation", "--remove-module"], + ["--resume-publication", "operation", "--managed"], + ] { + assert!(command + .clone() + .try_get_matches_from(std::iter::once("publish").chain(flags)) + .is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/publish/managed/environment_tests.rs b/crates/cli/src/subcommands/publish/managed/environment_tests.rs new file mode 100644 index 00000000000..582a5615234 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed/environment_tests.rs @@ -0,0 +1,239 @@ +use super::*; +use crate::container::publish::tests::{database, publisher, Fixture}; +use serde_json::json; +use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use std::collections::{BTreeMap, HashMap}; + +const KEY: &str = "STDB_MANAGED_FIXTURE_REQUIRED"; + +fn declared_module() -> deployment::system_empty::GeneratedModule { + deployment::system_empty::generate( + &EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: KEY.into(), + constraint: EnvironmentConstraint::OneOf(vec!["first-secret".into(), "second-secret".into()]), + optional: false, + }]) + .unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn keep_resolves_program_bound_declarations_and_retains_complete_input() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let module = declared_module(); + let deployment = deployment::DeploymentSpec::V1(deployment::DeploymentSpecV1 { + module: deployment::ModuleComponent::SystemEmpty(module.descriptor), + container: None, + }); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(deployment.revision().unwrap()), + deployment, + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + module_artifact: ArtifactReference { + digest: spacetimedb_oci::sha256(&module.bytes), + size_bytes: module.bytes.len() as u64, + }, + }; + fixture.state.lock().unwrap().selected_schema = Some(( + module.descriptor.program_hash, + crate::container::publish::tests::environment_tests::schema_bytes( + deployment::system_empty::verify(&module.descriptor, &module.bytes).unwrap(), + ), + )); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.try_get_matches_from(["publish", "--remove-container"]).unwrap(); + let target = CommandConfig::new( + &schema, + HashMap::from([("env".into(), json!({KEY:"first-secret"}))]), + &args, + ) + .unwrap(); + let journal = prepare_request( + &fixture.client(), + &target, + temporary.path(), + &temporary.path().join("state"), + Some(&prior), + None, + None, + publisher(), + &fixture.endpoint, + YesFlags::all(), + CancellationToken::new(), + ) + .await + .unwrap(); + let request = journal.record.request().unwrap(); + assert_eq!( + request.environment, + BTreeMap::from([(KEY.into(), "first-secret".into())]) + ); + assert_eq!( + request.manifest.current().envelope.expected_last_operation, + prior.last_operation + ); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Keep + )); + assert!(journal.record.uploads.is_empty()); + assert!(!std::fs::read_to_string(journal.directory().join("publication.json")) + .unwrap() + .contains("first-secret")); + let path = journal.directory().to_owned(); + let exact = journal.submission_bytes().unwrap(); + drop(journal); + // Neither the selected program endpoint nor mutable project input is read + // again after creation. The exact protected input drives recovery. + fixture.state.lock().unwrap().selected_schema = None; + std::fs::write( + temporary.path().join("spacetime.json"), + format!(r#"{{"env":{{"{KEY}":"second-secret"}}}}"#), + ) + .unwrap(); + let reopened = Journal::open(&path).unwrap(); + assert_eq!(reopened.submission_bytes().unwrap(), exact); + assert_eq!(fixture.state.lock().unwrap().module_gets, 1); + fixture.close().await; +} + +#[tokio::test] +async fn precompiled_exact_bytes_validate_required_invalid_unknown_and_explicit_values_before_mutation() { + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let module = declared_module(); + let wasm = temporary.path().join("exact.wasm"); + std::fs::write(&wasm, &module.bytes).unwrap(); + for (n, values, valid) in [ + (0, json!({}), false), + (1, json!({KEY:"secret-invalid-value"}), false), + (2, json!({KEY:"first-secret", "UNKNOWN":"secret-unknown-value"}), false), + (3, json!({KEY:"first-secret"}), true), + ] { + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "--bin-path", wasm.to_str().unwrap()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::from([("env".into(), values)]), &args).unwrap(); + let result = prepare_request( + &fixture.client(), + &target, + temporary.path(), + &temporary.path().join(format!("state{n}")), + None, + None, + None, + publisher(), + &fixture.endpoint, + YesFlags::all(), + CancellationToken::new(), + ) + .await; + if valid { + let journal = result.unwrap(); + assert_eq!( + std::fs::read(journal.directory().join("module.blob")).unwrap(), + module.bytes.as_ref() + ); + assert_eq!(journal.record.request().unwrap().environment[KEY], "first-secret"); + } else { + let error = format!("{:#}", result.err().unwrap()); + assert!(!error.contains("secret-invalid-value") && !error.contains("secret-unknown-value")); + assert!(!temporary.path().join(format!("state{n}")).exists()); + } + } + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn fresh_process_resume_replays_original_values_after_shell_changes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = crate::container::publish::tests::temporary_directory(); + let mut record = fixture.record(false, false); + let mut request = record.request().unwrap(); + request.environment.insert(KEY.into(), "first-secret".into()); + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::system_empty::empty().bytes.as_ref()), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let directory = journal.directory().to_owned(); + drop(journal); + let marker = temporary.path().join("child-finished"); + let mut child = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "subcommands::publish::managed::environment_tests::resume_changed_shell_child", + ]) + .env_clear() + .env("STDB_MANAGED_FIXTURE_RESUME", &directory) + .env("STDB_MANAGED_FIXTURE_MARKER", &marker) + .env(KEY, "second-secret") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let result = tokio::time::timeout(Duration::from_secs(20), child.wait()).await; + let status = match result { + Ok(status) => status.unwrap(), + Err(error) => { + child.start_kill().unwrap(); + child.wait().await.unwrap(); + panic!("resume child exceeded its deadline: {error}"); + } + }; + assert!(status.success()); + assert_eq!(std::fs::read(&marker).unwrap(), b"one fixture completed"); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; +} + +#[tokio::test] +#[ignore = "invoked only by the owned fresh-process resume fixture"] +async fn resume_changed_shell_child() { + assert_eq!(std::env::var(KEY).unwrap(), "second-secret"); + let directory = PathBuf::from(std::env::var_os("STDB_MANAGED_FIXTURE_RESUME").unwrap()); + let marker = PathBuf::from(std::env::var_os("STDB_MANAGED_FIXTURE_MARKER").unwrap()); + assert!(directory.is_absolute() && marker.is_absolute()); + let mut journal = Journal::open(&directory).unwrap(); + assert!(journal.record.server.starts_with("http://127.0.0.1:")); + let client = PublisherClient::new( + &journal.record.server, + "Bearer isolated-fixture-credential".parse().unwrap(), + ) + .unwrap(); + publish::run(&client, &mut journal, None, Duration::ZERO, CancellationToken::new()) + .await + .unwrap(); + assert_eq!(journal.record.request().unwrap().environment[KEY], "first-secret"); + drop(journal); + std::fs::write(marker, b"one fixture completed").unwrap(); +} diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 8b26700167e..ce542a2a43f 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -324,14 +324,14 @@ enum Error { #[error("error sending subscription queries")] Subscribe { #[source] - source: WsError, + source: Box, }, #[error("protocol error: {details}")] Protocol { details: &'static str }, #[error("websocket error: {source}")] Websocket { #[source] - source: WsError, + source: Box, }, #[error("encountered failed transaction: {reason}")] TransactionFailure { reason: Box }, @@ -384,12 +384,7 @@ impl UpdateCounter { impl Error { fn is_server_closed_connection(&self) -> bool { - matches!( - self, - Self::Websocket { - source: WsError::ConnectionClosed - } - ) + matches!(self, Self::Websocket { source } if matches!(source.as_ref(), WsError::ConnectionClosed)) } } @@ -403,7 +398,7 @@ fn connection_closed_error(num: Option, num_received: u32) -> Error { None => { eprintln!("disconnected by server"); Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), } } } @@ -421,7 +416,9 @@ where }, ))) .unwrap(); - ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) + ws.send(msg.into()) + .await + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Send a v3 BSATN subscribe message. @@ -437,7 +434,7 @@ where let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; ws.send(WsMessage::Binary(msg.into())) .await - .map_err(|source| Error::Subscribe { source }) + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Parse a v1 text websocket message as JSON. @@ -457,7 +454,11 @@ where { const RECV_TX_UPDATE: &str = "protocol error: received transaction update before initial subscription update"; - while let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? { + while let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + { let Some(msg) = parse_msg_json(&msg) else { continue }; match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { @@ -542,7 +543,11 @@ where if num.is_some_and(|n| num_received.get() >= n) { return Ok(()); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Err(connection_closed_error(num, num_received.get())); }; @@ -627,7 +632,11 @@ where return Ok(Some(msg)); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Ok(None); }; let WsMessage::Binary(msg) = msg else { continue }; diff --git a/crates/cli/tests/container_build_acceptance/README.md b/crates/cli/tests/container_build_acceptance/README.md new file mode 100644 index 00000000000..fd8954c0f07 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/README.md @@ -0,0 +1,77 @@ +# Real local container builder acceptance + +This opt-in fixture exercises the CLI with actual BuildKit and explicitly +selected Railpack. It does not connect to a SpacetimeDB server. The resulting +verified OCI layouts can be consumed by the separate managed publication +acceptance test. + +The checked-in tool lock currently supports macOS on arm64. It records official +release download URLs, SHA-256 checksums, and image digests. The fixture downloads +the two tools into its own workspace and extracts only the expected regular +binary. Nothing is installed globally. Release checksum provenance is recorded +in `tool-lock.json`; image pins were verified against the primary registries. + +Build the CLI and deadline harness from the public workspace, without selecting +or connecting to a server: + +```sh +cargo build --locked --offline -p spacetimedb-cli --bin spacetimedb-cli +cargo test --locked --offline -p spacetimedb-cli --test real_container_builder --no-run +``` + +Use the absolute test executable path printed by the second command. Invoke the +script with absolute paths and a new workspace directory: + +```sh +python3 crates/cli/tests/container_build_acceptance/acceptance.py \ + --docker-socket /absolute/path/to/verified/docker-desktop.sock \ + --cli /absolute/path/to/public/target/debug/spacetimedb-cli \ + --deadline-test-binary /absolute/path/to/public/target/debug/deps/real_container_builder-HASH \ + --workspace /private/tmp/new-owned-builder-workspace +``` + +The socket must already be verified as an owned disposable Docker Desktop +endpoint. The script checks its reported host identity again. Every Docker +command specifies that socket and an empty fixture configuration; saved Docker +contexts and registry credentials are not used. The fixture pulls pinned public +images, starts a uniquely named BuildKit container with a dedicated cache volume, +and binds its port to numeric loopback. A private Unix socket forwards only to +that port. The container is limited to two CPUs, 2 GiB of memory and 256 processes. +Container logs rotate at 4 MiB with at most two files. BuildKit requires +privileged execution in this local fixture; the Docker socket +is not mounted into it. This is a trusted-tool test, not a sandbox for arbitrary +build programs. + +The checks cover: + +- A Dockerfile using a build-secret mount and an explicitly selected Railpack + shell-script build. Neither successful nor failed build logs may reveal the + secret, and retained image objects must not contain it. +- Exact digest, size, platform and executable manifest/config/layer closure of + both prepared outputs. A modified retained object is rejected on import. +- Failed builds and failed Railpack detection, with no automatic builder fallback. +- A destination created while a build is running, which must remain untouched. +- SIGINT cancellation of real `buildctl`, followed by positive PID absence and + workspace release. +- The same cleanup on a two-second deadline through the production local process + runner. Only the test runner shortens the normal build deadline. + +Commands have finite deadlines. Teardown stops the proxy, validates the exact +owned container's name and label, waits for its successful synchronous removal, +then removes its cache volume. If the run reply is lost, cleanup looks up only +the original generated name and requires the same ownership label. An ambiguous +daemon answer fails cleanup; an arbitrary inspection error is not treated as +proof of absence. A teardown failure fails the fixture. Downloaded binaries, +diagnostic inputs and the two OCI layouts remain in the private workspace; public image cache entries +are not pruned. `acceptance.json` records the completed checks and the output +paths without credentials or build-secret values. + +The retained layouts establish build and artifact preparation behavior. They do +not establish container execution, readiness, or isolation in the production +runtime. Those are separate Linux supervisor and Kata acceptance boundaries. + +The cleanup failure tests need no Docker daemon or network: + +```sh +python3 -B -m unittest discover -s crates/cli/tests/container_build_acceptance -p test_fixture.py -v +``` diff --git a/crates/cli/tests/container_build_acceptance/acceptance.py b/crates/cli/tests/container_build_acceptance/acceptance.py new file mode 100644 index 00000000000..ebbc575dd6c --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/acceptance.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Explicitly invoked real local-builder acceptance, never part of cargo test. + +Only an explicitly verified Docker Desktop Unix socket is accepted. The CLI +uses a fixture-owned BuildKit Unix proxy and never opens Spacetime credentials. +Downloaded tools and retained OCI output live under a new private workspace. +""" +import argparse +import concurrent.futures +import gzip +import hashlib +import json +import os +from pathlib import Path +import platform +import selectors +import shutil +import shlex +import signal +import socket +import subprocess +import tarfile +import threading +import time +import urllib.request +import uuid + + +LOCK = json.loads(Path(__file__).with_name("tool-lock.json").read_text()) +LIMIT = 32 * 1024 * 1024 + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def environment(root): + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + } + + +def run(args, env, timeout=120, check=True): + # No shell expansion, inherited proxy/daemon/server variables or credentials. + child = subprocess.Popen(args, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = child.communicate(timeout=timeout) + except BaseException: + child.kill() + child.communicate(timeout=10) + raise + require(len(out) <= LIMIT and len(err) <= LIMIT, "fixture command output exceeded bound") + if check: + require(child.returncode == 0, + f"fixture command failed: {Path(args[0]).name} (status {child.returncode})") + return child.returncode, out, err + + +def tools(root): + require(platform.system() == "Darwin" and platform.machine() == "arm64", + "the checked-in tool lock supports Darwin arm64") + directory = root / "tools" + directory.mkdir() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + binaries = {} + for name, pin in LOCK["tools"].items(): + print(f"Downloading and verifying pinned {name} {pin['version']}", flush=True) + with opener.open(pin["url"], timeout=60) as response: + data = response.read(LIMIT + 1) + require(len(data) <= LIMIT, "tool archive exceeded bound") + require(hashlib.sha256(data).hexdigest() == pin["sha256"], "tool archive checksum mismatch") + archive = directory / (name + ".tar.gz") + archive.write_bytes(data) + with tarfile.open(archive) as contents: + matches = [item for item in contents if item.name.removeprefix("./") == pin["member"]] + require(len(matches) == 1 and matches[0].isreg(), "unexpected tool archive member") + require(matches[0].size <= 128 * 1024 * 1024, "tool binary exceeded bound") + binary = directory / name + with contents.extractfile(matches[0]) as source: + binary.write_bytes(source.read()) + binary.chmod(0o700) + binaries[name] = binary + return binaries + + +class UnixProxy: + """Forward only to this fixture's Docker-published numeric-loopback port.""" + def __init__(self, path, port): + self.path, self.port = path, port + self.stop = threading.Event() + self.slots = threading.BoundedSemaphore(32) + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=32) + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(str(path)) + path.chmod(0o600) + self.listener.listen(32) + self.listener.settimeout(0.2) + self.thread = threading.Thread(target=self.accept) + self.thread.start() + + def accept(self): + while not self.stop.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + break + if self.slots.acquire(blocking=False): + self.pool.submit(self.forward, client) + else: + client.close() + + def forward(self, client): + try: + with client, socket.create_connection(("127.0.0.1", self.port), timeout=5) as upstream: + # Bounded socket writes and periodic stop checks; no filesystem + # path or request header can select another destination. + client.settimeout(1) + upstream.settimeout(1) + with selectors.DefaultSelector() as select: + select.register(client, selectors.EVENT_READ, upstream) + select.register(upstream, selectors.EVENT_READ, client) + while not self.stop.is_set(): + for key, _ in select.select(0.2): + data = key.fileobj.recv(64 * 1024) + if not data: + return + key.data.sendall(data) + except (OSError, TimeoutError): + pass + finally: + self.slots.release() + + def close(self): + self.stop.set() + self.listener.close() + self.thread.join(timeout=3) + require(not self.thread.is_alive(), "BuildKit proxy did not stop accepting") + self.pool.shutdown(wait=True) + self.path.unlink() + + +class Builder: + def __init__(self, root, docker_socket, binaries): + self.root, self.binaries = root, binaries + self.env = environment(root) + docker = shutil.which("docker") + require(docker is not None, "Docker executable is required") + self.docker = [docker, "--host", "unix://" + str(docker_socket), + "--config", str(root / "docker-config")] + self.name = "stdb-cli-build-" + uuid.uuid4().hex + self.volume = self.name + "-cache" + self.container = None + self.run_attempted = False + self.proxy = None + self.volume_created = False + + def command(self, *args, timeout=120, check=True): + return run(self.docker + list(args), self.env, timeout, check) + + def start(self): + _, info, _ = self.command("info", "--format", "{{.Name}}|{{.OperatingSystem}}|{{.OSType}}") + require(info.decode().strip() == "docker-desktop|Docker Desktop|linux", + "explicit socket is not the expected local Docker Desktop fixture host") + print("Verified explicit Docker Desktop socket; starting isolated BuildKit", flush=True) + self.command("pull", "--platform", "linux/arm64", LOCK["images"]["buildkit"], timeout=300) + _, raw, _ = self.command("image", "inspect", LOCK["images"]["buildkit"]) + inspected = json.loads(raw)[0] + require(LOCK["images"]["buildkit"] in inspected["RepoDigests"], "pulled BuildKit digest mismatch") + require(inspected["Architecture"] == "arm64", "BuildKit architecture mismatch") + self.command("volume", "create", "--label", "spacetimedb.fixture=" + self.name, self.volume) + self.volume_created = True + # A timed-out CLI may have created the container before losing its + # response. Retain the exact name/label cleanup key before dispatch. + self.run_attempted = True + _, identifier, _ = self.command( + "run", "--detach", "--pull", "never", "--platform", "linux/arm64", + "--name", self.name, "--label", "spacetimedb.fixture=" + self.name, + "--privileged", "--cpus", "2", "--memory", "2g", "--memory-swap", "2g", + "--log-driver", "json-file", "--log-opt", "max-size=4m", "--log-opt", "max-file=2", + "--pids-limit", "256", "--mount", f"type=volume,src={self.volume},dst=/var/lib/buildkit", + "--publish", "127.0.0.1::1234", LOCK["images"]["buildkit"], + "--addr", "tcp://0.0.0.0:1234", "--oci-worker-snapshotter=native") + self.container = identifier.decode().strip() + _, raw, _ = self.command("inspect", self.container) + owned = json.loads(raw)[0] + require(owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, "container ownership mismatch") + ports = owned["NetworkSettings"]["Ports"]["1234/tcp"] + require(len(ports) == 1 and ports[0]["HostIp"] == "127.0.0.1", "BuildKit is not loopback bound") + self.proxy = UnixProxy(self.root / "buildkit.sock", int(ports[0]["HostPort"])) + deadline = time.monotonic() + 30 + while True: + status, _, _ = run([str(self.binaries["buildctl"]), "--addr", self.endpoint, + "debug", "workers"], self.env, timeout=5, check=False) + if status == 0: + break + require(time.monotonic() < deadline, "BuildKit fixture did not become ready") + time.sleep(0.2) + + @property + def endpoint(self): + return "unix://" + str(self.root / "buildkit.sock") + + def close(self): + errors = [] + if self.proxy: + try: + self.proxy.close() + except Exception as error: + errors.append(str(error)) + if self.run_attempted: + try: + # Inspect only the exact returned ID or generated name. A + # failed/ambiguous inspect is not proof of absence. The label + # must match before deletion, including after a lost run reply. + _, raw, _ = self.command("container", "inspect", self.container or self.name) + objects = json.loads(raw) + require(len(objects) == 1, "ambiguous owned container lookup") + owned = objects[0] + require(owned["Name"] == "/" + self.name and + owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, + "container cleanup ownership mismatch") + if self.container: + require(owned["Id"] == self.container, "container cleanup ID mismatch") + # A successful synchronous removal is positive completion. + self.command("rm", "--force", "--volumes", owned["Id"]) + except Exception as error: + errors.append(str(error)) + if self.volume_created: + try: + self.command("volume", "rm", self.volume) + except Exception as error: + errors.append(str(error)) + require(not errors, "positive builder teardown failed: " + "; ".join(errors)) + + +def config(project, image): + project.mkdir() + document = { + "database": "local-builder-fixture", + "container": { + "image": image, + "env_keys": ["RUNTIME_ONLY"], + "resources": {"cpu_millicores": 1000, "memory_bytes": 536870912, + "scratch_bytes": 1073741824, "pids_max": 128}, + }, + } + (project / "spacetime.json").write_text(json.dumps(document)) + + +def verify_layout(layout, secret): + metadata = json.loads((layout / "prepared.json").read_text()) + require(secret not in (layout / "prepared.json").read_bytes(), "build secret entered prepared metadata") + objects = {} + for item in metadata["objects"]: + descriptor = item["descriptor"] + path = Path(item["path"]) + require(not path.is_absolute() and ".." not in path.parts, "artifact path escaped layout") + data = (layout / path).read_bytes() + require(len(data) == descriptor["size"], "artifact size mismatch") + require("sha256:" + hashlib.sha256(data).hexdigest() == descriptor["digest"], "artifact hash mismatch") + require(secret not in data, "build secret entered retained image object") + if item["kind"] == "layer" and data[:2] == b"\x1f\x8b": + require(secret not in gzip.decompress(data), "build secret entered retained image layer") + objects[descriptor["digest"]] = (item["kind"], data) + manifest = json.loads(objects[metadata["manifest"]["digest"]][1]) + require(set(objects) == {metadata["manifest"]["digest"], manifest["config"]["digest"], + *(entry["digest"] for entry in manifest["layers"])}, + "prepared closure is not the exact executable manifest/config/layers") + image = json.loads(objects[manifest["config"]["digest"]][1]) + require(image["os"] == "linux" and image["architecture"] == "arm64", "image platform mismatch") + return metadata, image + + +def cases(root, cli, binaries, builder, deadline_test): + secret = ("build-secret-" + uuid.uuid4().hex).encode() + secret_file = root / "build-secret" + secret_file.write_bytes(secret) + secret_file.chmod(0o600) + base = [str(cli), "--root-dir", str(root / "cli-root"), "--config-path", str(root / "unused-config"), + "container", "build", "--platform", "linux/arm64", "--buildctl", str(binaries["buildctl"]), + "--railpack", str(binaries["railpack"]), "--buildkit-host", builder.endpoint] + + def build(project, output, success=True, secrets=False): + args = base + ["--project-path", str(project), "--out-dir", str(output)] + if secrets: + args += ["--build-secret", "BUILD_SENTINEL=" + str(secret_file)] + status, out, err = run(args, environment(root), timeout=1200, check=False) + require(secret not in out + err, "CLI leaked builder secret diagnostics") + if (status == 0) != success: + # Error output has been checked for the generated sentinel and the + # fixture never supplies ordinary or registry credentials. + raise RuntimeError("CLI build outcome mismatch: " + (out + err).decode(errors="replace")[:2048]) + return out + err + + project = root / "dockerfile-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL test -s /run/secrets/BUILD_SENTINEL && cat /run/secrets/BUILD_SENTINEL >&2\n" + "WORKDIR /app\nUSER 1001:1001\nENV IMAGE_DEFAULT=retained\n" + 'ENTRYPOINT ["/bin/sh"]\nCMD ["-c", "echo dockerfile-ready"]\n') + output = root / "dockerfile-output" + print("Actual Dockerfile build with explicit secret mount", flush=True) + build(project, output, secrets=True) + metadata, image = verify_layout(output, secret) + require(metadata["container"]["argv"] == ["/bin/sh", "-c", "echo dockerfile-ready"], "argv was not normalized") + require(metadata["container"]["user"] == "1001:1001", "image user was lost") + require(metadata["container"]["working_directory"] == "/app", "image working directory was lost") + require("IMAGE_DEFAULT=retained" in image["config"]["Env"], "image environment defaults were lost") + + project = root / "railpack-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "start.sh").write_text("#!/bin/sh\nprintf 'railpack-ready\\n'\n") + (project / "start.sh").chmod(0o700) + output = root / "railpack-output" + print("Actual explicitly selected Railpack build", flush=True) + build(project, output) + verify_layout(output, secret) + + print("Failed secret-using build must not expose logs or publish output", flush=True) + project = root / "failed-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL cat /run/secrets/BUILD_SENTINEL >&2; exit 23\n") + output = root / "failed-output" + build(project, output, success=False, secrets=True) + require(not output.exists(), "failed build published output") + + print("Explicit unsupported Railpack detection cannot fall back to Dockerfile", flush=True) + project = root / "unsupported-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "Dockerfile").write_text('FROM scratch\nCMD ["would-be-wrong-builder"]\n') + output = root / "unsupported-output" + build(project, output, success=False) + require(not output.exists(), "unsupported Railpack silently used another builder") + + print("Tampering with a retained OCI object must fail before output publication", flush=True) + tampered = root / "tampered-layout" + shutil.copytree(root / "dockerfile-output", tampered) + metadata = json.loads((tampered / "prepared.json").read_text()) + target = next(item for item in metadata["objects"] if item["kind"] == "config") + with (tampered / target["path"]).open("ab") as changed: + changed.write(b" ") + project = root / "tampered-project" + config(project, {"oci_ref": "oci:" + str(tampered)}) + output = root / "tampered-output" + build(project, output, success=False) + require(not output.exists(), "tampered OCI image was accepted") + + print("A path created during the actual build must never be replaced", flush=True) + project = root / "replacement-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 3\nCMD [\"/bin/true\"]\n") + output = root / "replacement-output" + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task = pool.submit(build, project, output, False) + deadline = time.monotonic() + 30 + while not list(root.glob(".spacetime-image-*")): + require(not task.done(), "build finished before concurrent replacement could be tested") + require(time.monotonic() < deadline, "build never created its private workspace") + time.sleep(0.01) + output.mkdir() + (output / "sentinel").write_text("must survive") + task.result(timeout=60) + require(list(output.iterdir()) == [output / "sentinel"], "existing output was replaced") + require((output / "sentinel").read_text() == "must survive", "existing output content changed") + + print("SIGINT must cancel the real buildctl and positively reap it", flush=True) + project = root / "cancel-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 120\nCMD [\"/bin/true\"]\n") + output = root / "cancel-output" + pid_file = root / "actual-buildctl.pid" + wrapper = root / "record-buildctl" + wrapper.write_text("#!/bin/sh\nprintf '%s\\n' \"$$\" > " + shlex.quote(str(pid_file)) + + "\nexec " + shlex.quote(str(binaries["buildctl"])) + ' "$@"\n') + wrapper.chmod(0o700) + args = list(base) + args[args.index("--buildctl") + 1] = str(wrapper) + args += ["--project-path", str(project), "--out-dir", str(output)] + child = subprocess.Popen(args, env=environment(root), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 30 + while not pid_file.exists(): + require(child.poll() is None, "CLI exited before invoking real buildctl") + require(time.monotonic() < deadline, "real buildctl did not start") + time.sleep(0.01) + tool_pid = int(pid_file.read_text()) + child.send_signal(signal.SIGINT) + out, err = child.communicate(timeout=15) + require(child.returncode != 0 and b"cancelled" in out + err, "CLI did not report cancellation") + require(secret not in out + err, "cancelled CLI leaked a secret") + try: + os.kill(tool_pid, 0) + except ProcessLookupError: + pass + else: + raise RuntimeError("real buildctl PID still exists after acknowledged CLI cancellation") + require(not output.exists(), "cancelled build published output") + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=10) + require(not list(root.glob(".spacetime-image-*")), "completed CLI left builder workspaces behind") + + print("A short harness deadline must reap real buildctl before releasing its workspace", flush=True) + pid_file.unlink() + deadline_env = environment(root) + deadline_env.update({ + "STDB_BUILDER_CONTEXT": str(project), + "STDB_BUILDER_BUILDCTL": str(wrapper), + "STDB_BUILDER_PID_FILE": str(pid_file), + "STDB_BUILDER_WORKSPACE": str(root), + "STDB_BUILDER_SOCKET": builder.endpoint, + }) + run([str(deadline_test), "--ignored", "--exact", + "actual_buildctl_deadline_reaps_before_workspace_release", "--test-threads=1"], + deadline_env, timeout=30) + return { + "version": 1, + "dockerfile": str(root / "dockerfile-output"), + "railpack": str(root / "railpack-output"), + "tool_lock": LOCK, + "checks": ["actual_dockerfile", "actual_railpack", "secret_logs", "failed_build", + "no_fallback", "tampered_closure", "concurrent_output", "cancel_and_reap", + "deadline_and_reap"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docker-socket", type=Path, required=True) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--deadline-test-binary", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + require(args.docker_socket.is_absolute() and args.docker_socket.is_socket(), "explicit Docker Unix socket required") + require(args.cli.is_absolute() and args.cli.is_file(), "absolute prebuilt CLI executable required") + require(args.deadline_test_binary.is_absolute() and args.deadline_test_binary.is_file(), + "absolute prebuilt deadline test executable required") + require(args.workspace.is_absolute() and not args.workspace.exists(), "workspace must be a new absolute directory") + args.workspace.mkdir(mode=0o700) + for child in ["home", "tmp", "docker-config"]: + (args.workspace / child).mkdir(mode=0o700) + # A missing named config proves local build dispatch does not read global + # credentials or open the supplied ordinary configuration path. + binaries = tools(args.workspace) + builder = Builder(args.workspace, args.docker_socket, binaries) + try: + builder.start() + receipt = cases(args.workspace, args.cli, binaries, builder, args.deadline_test_binary) + finally: + builder.close() + (args.workspace / "acceptance.json").write_text(json.dumps(receipt, indent=2)) + print("Builder acceptance passed; retained OCI layouts are ready for managed publication", flush=True) + + +if __name__ == "__main__": + main() diff --git a/crates/cli/tests/container_build_acceptance/test_fixture.py b/crates/cli/tests/container_build_acceptance/test_fixture.py new file mode 100644 index 00000000000..40ed63ea94d --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/test_fixture.py @@ -0,0 +1,64 @@ +"""Owned-resource failure paths, without contacting Docker or any server.""" +import json +import unittest + +from acceptance import Builder + + +class CleanupTests(unittest.TestCase): + def builder(self, reply): + builder = Builder.__new__(Builder) + builder.name = "fixture-owned-name" + builder.container = None # docker run reply was lost + builder.run_attempted = True + builder.proxy = None + builder.volume_created = False + calls = [] + + def command(*args): + calls.append(args) + if args[:2] == ("container", "inspect"): + if isinstance(reply, Exception): + raise reply + return 0, json.dumps(reply).encode(), b"" + return 0, b"", b"" + + builder.command = command + return builder, calls + + def owned(self): + return [{"Name": "/fixture-owned-name", "Id": "owned-id", "Config": { + "Labels": {"spacetimedb.fixture": "fixture-owned-name"}}}] + + def test_lost_run_reply_removes_only_exact_owned_container(self): + builder, calls = self.builder(self.owned()) + builder.close() + self.assertEqual(calls, [ + ("container", "inspect", "fixture-owned-name"), + ("rm", "--force", "--volumes", "owned-id"), + ]) + + def test_wrong_label_never_authorizes_removal(self): + reply = self.owned() + reply[0]["Config"]["Labels"]["spacetimedb.fixture"] = "another-fixture" + builder, calls = self.builder(reply) + with self.assertRaisesRegex(RuntimeError, "ownership mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_inspect_error_is_incomplete_cleanup(self): + builder, calls = self.builder(RuntimeError("daemon unavailable")) + with self.assertRaisesRegex(RuntimeError, "positive builder teardown failed"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_returned_identifier_must_match_inspected_object(self): + builder, calls = self.builder(self.owned()) + builder.container = "different-id" + with self.assertRaisesRegex(RuntimeError, "ID mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/cli/tests/container_build_acceptance/tool-lock.json b/crates/cli/tests/container_build_acceptance/tool-lock.json new file mode 100644 index 00000000000..e62b8e64ec2 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/tool-lock.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "platform": "darwin-arm64", + "tools": { + "buildctl": { + "version": "0.33.0", + "url": "https://github.com/moby/buildkit/releases/download/v0.33.0/buildkit-v0.33.0.darwin-arm64.tar.gz", + "sha256": "730ac4ffd6f4a88dc404fc675aeaf4cfee414915036042847f0a60861cc8790c", + "member": "bin/buildctl", + "source": "https://api.github.com/repos/moby/buildkit/releases/tags/v0.33.0" + }, + "railpack": { + "version": "0.35.0", + "url": "https://github.com/railwayapp/railpack/releases/download/v0.35.0/railpack-v0.35.0-arm64-apple-darwin.tar.gz", + "sha256": "fb4c16d57458eb7868d48ed8a454014ef40716a6c939929d7b7d5986563a0c65", + "member": "railpack", + "source": "https://api.github.com/repos/railwayapp/railpack/releases/tags/v0.35.0" + } + }, + "images": { + "buildkit": "moby/buildkit@sha256:6c2fa84a6b61ccd72899dde4239f8d5717f05f9a8ca6f3cad185fb1a95a94de3", + "buildkit_arm64": "sha256:e8efce994e456acb94944bcc0530b3188478572914f413d4476568d8b63515c6", + "alpine": "alpine@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1", + "railpack_frontend": "ghcr.io/railwayapp/railpack-frontend@sha256:bc73534934e7929ab3dc41765fb7e25c8c69d9be98c43ef8792fea51f65317bd" + }, + "image_sources": [ + "https://registry-1.docker.io/v2/moby/buildkit/manifests/v0.33.0", + "https://registry-1.docker.io/v2/library/alpine/manifests/3.22.1", + "https://ghcr.io/v2/railwayapp/railpack-frontend/manifests/v0.35.0" + ] +} diff --git a/crates/cli/tests/real_container_builder.rs b/crates/cli/tests/real_container_builder.rs new file mode 100644 index 00000000000..cad32c9c63b --- /dev/null +++ b/crates/cli/tests/real_container_builder.rs @@ -0,0 +1,80 @@ +//! Run only from container_build_acceptance/acceptance.py, which owns and +//! verifies the Docker/BuildKit endpoint and all paths below. No saved defaults. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use anyhow::{ensure, Context, Result}; +use spacetimedb_cli::container::{ + config::ContainerConfig, + prepare_container, + process::{Invocation, LocalRunner, Output, Runner}, + BuildTools, +}; +use spacetimedb_lib::container::ImagePlatform; +use std::{path::PathBuf, time::Duration}; +use tokio_util::sync::CancellationToken; + +struct ShortDeadline; +impl Runner for ShortDeadline { + async fn run(&self, mut invocation: Invocation) -> Result { + // Exercise the production local owner/kill/reap path without making + // acceptance wait for the ordinary thirty-minute build deadline. + invocation.timeout = Duration::from_secs(2); + LocalRunner.run(invocation).await + } +} + +fn input(name: &str) -> Result { + let path = PathBuf::from(std::env::var_os(name).with_context(|| format!("explicit {name} is required"))?); + ensure!(path.is_absolute(), "fixture input must be absolute"); + Ok(path) +} + +#[tokio::test] +#[ignore = "requires the explicitly owned local BuildKit acceptance fixture"] +async fn actual_buildctl_deadline_reaps_before_workspace_release() -> Result<()> { + let context = input("STDB_BUILDER_CONTEXT")?.canonicalize()?; + let tool = input("STDB_BUILDER_BUILDCTL")?.canonicalize()?; + let pid_file = input("STDB_BUILDER_PID_FILE")?; + let workspace = input("STDB_BUILDER_WORKSPACE")?.canonicalize()?; + let endpoint = std::env::var("STDB_BUILDER_SOCKET")?; + ensure!( + endpoint.starts_with("unix:///"), + "explicit local BuildKit Unix socket is required" + ); + let document: serde_json::Value = serde_json::from_slice(&std::fs::read(context.join("spacetime.json"))?)?; + let configuration: ContainerConfig = serde_json::from_value(document["container"].clone())?; + let result = prepare_container( + &configuration, + &context, + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + &BuildTools { + buildctl: tool, + buildkit_host: Some(endpoint), + ..Default::default() + }, + &workspace, + &ShortDeadline, + CancellationToken::new(), + ) + .await; + let error = result.err().context("long-running real build unexpectedly succeeded")?; + ensure!( + error.to_string().contains("build deadline"), + "unexpected build failure: {error:#}" + ); + let pid = std::fs::read_to_string(pid_file)?.trim().parse()?; + let pid = rustix::process::Pid::from_raw(pid).context("invalid recorded builder PID")?; + ensure!( + rustix::process::test_kill_process(pid) == Err(rustix::io::Errno::SRCH), + "real buildctl PID still exists after the deadline returned" + ); + ensure!( + std::fs::read_dir(workspace)?.all(|entry| entry + .is_ok_and(|entry| !entry.file_name().to_string_lossy().starts_with(".spacetime-image-"))), + "deadline returned before its owned workspace was released" + ); + Ok(()) +} diff --git a/crates/client-api/src/auth.rs b/crates/client-api/src/auth.rs index 729d001e2c3..38d91489c7e 100644 --- a/crates/client-api/src/auth.rs +++ b/crates/client-api/src/auth.rs @@ -88,6 +88,7 @@ pub struct SpacetimeAuth { pub claims: SpacetimeIdentityClaims, /// The JWT payload as a json string (after base64 decoding). pub jwt_payload: Box, + pub hosted: Option, } impl SpacetimeAuth { @@ -100,6 +101,7 @@ impl SpacetimeAuth { creds, claims, jwt_payload: payload, + hosted: None, }) } } @@ -109,6 +111,7 @@ impl From for ConnectionAuthCtx { ConnectionAuthCtx { claims: auth.claims, jwt_payload: auth.jwt_payload.clone(), + hosted: auth.hosted, } } } @@ -214,6 +217,9 @@ impl SpacetimeAuth { signer: &impl TokenSigner, expiry: Duration, ) -> Result<(SpacetimeIdentityClaims, String), JwtError> { + if self.hosted.is_some() { + return Err(JwtErrorKind::InvalidToken.into()); + } TokenClaims::from(self.clone()).encode_and_sign_with_expiry(signer, Some(expiry)) } } @@ -402,13 +408,47 @@ pub struct SpacetimeAuthHeader { } #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthHeader { +impl axum::extract::FromRequestParts for SpacetimeAuthHeader { type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let Some(creds) = SpacetimeCreds::from_request_parts(parts)? else { return Ok(Self { auth: None }); }; + if spacetimedb::auth::hosted_tokens::has_reserved_hosted_token_kind(&creds.token) + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))? + { + // Use Axum's matched route parameters, never a client header or a + // hand-parsed URL suffix. Tokens are ineligible for root publishing, + // identity allocation, and generic token exchange routes. + #[derive(Deserialize)] + struct HostedTargetPath { + name_or_identity: crate::util::NameOrIdentity, + } + let axum::extract::Path(params) = axum::extract::Path::::from_request_parts(parts, state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let target = params + .name_or_identity + .resolve(state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let verified = state + .authenticate_hosted_token(&creds.token, target) + .await + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let connection = verified + .into_connection_auth() + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let auth = SpacetimeAuth { + creds, + claims: connection.claims, + jwt_payload: connection.jwt_payload, + hosted: connection.hosted, + }; + return Ok(Self { auth: Some(auth) }); + } + let claims = validate_token(state, &creds.token) .await .map_err(AuthorizationRejection::Custom)?; @@ -420,6 +460,7 @@ impl axum::extract::FromRequestParts for Space creds, claims, jwt_payload: payload.into(), + hosted: None, }; Ok(Self { auth: Some(auth) }) } @@ -478,7 +519,9 @@ impl SpacetimeAuthHeader { pub struct SpacetimeAuthRequired(pub SpacetimeAuth); #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthRequired { +impl axum::extract::FromRequestParts + for SpacetimeAuthRequired +{ type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let auth = SpacetimeAuthHeader::from_request_parts(parts, state).await?; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7d3175e751b..ea941dd9c28 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -55,6 +55,15 @@ pub trait NodeDelegate: Send + Sync { type JwtAuthProviderT: auth::JwtAuthProvider; fn jwt_auth_provider(&self) -> &Self::JwtAuthProviderT; + /// Authenticate a platform-issued hosted credential for the exact resolved + /// database in this request. Editions without container hosting fail closed. + async fn authenticate_hosted_token( + &self, + _token: &str, + _target: Identity, + ) -> anyhow::Result { + anyhow::bail!("Hosted database credentials are not enabled on this server") + } /// Return the leader [`Host`] of `database_id`. /// /// The [`Host`] is spawned implicitly if not already running. @@ -136,7 +145,7 @@ impl Host { pub async fn exec_sql( &self, - auth: AuthCtx, + auth: impl Into, _database: Database, confirmed_read: bool, body: String, @@ -226,6 +235,52 @@ impl Host { ) .await } + + /// Commit the complete environment and admitted deployment in one host transaction. + #[allow(clippy::too_many_arguments)] + pub async fn update_with_environment_and_deployment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + deployment: spacetimedb::db::deployment::DeploymentCommit, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_environment_and_deployment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + environment, + Some(deployment), + ) + .await + } + + /// Used only by an authenticated publication coordinator after control + /// admission and quiescing. This does not authorize or start a container. + pub async fn update_with_deployment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: spacetimedb::db::deployment::DeploymentCommit, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_deployment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + Some(deployment), + ) + .await + } } /// Parameters for publishing a database. /// @@ -517,6 +572,14 @@ impl NodeDelegate for Arc { (**self).jwt_auth_provider() } + async fn authenticate_hosted_token( + &self, + token: &str, + target: Identity, + ) -> anyhow::Result { + (**self).authenticate_hosted_token(token, target).await + } + async fn leader(&self, database_id: u64) -> Result { (**self).leader(database_id).await } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 1aa2458074f..46c386d225b 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -168,13 +168,14 @@ pub async fn call( let caller_auth: ConnectionAuthCtx = auth.into(); + let caller = spacetimedb::auth::invocation::InvocationCaller::from(&caller_auth); let owner_identity = database.owner_identity; let module = find_database_module(&worker_ctx, &database).await?; - let fut = async move |module: ModuleHost, caller_identity: Identity, connection_id: ConnectionId| { + let fut = async move |module: ModuleHost, _caller_identity: Identity, connection_id: ConnectionId| { let result = match module .call_reducer( - caller_identity, + caller.clone(), Some(connection_id), None, None, @@ -188,7 +189,7 @@ pub async fn call( Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { // Not a reducer — try procedure instead match module - .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) + .call_procedure(caller, Some(connection_id), None, &reducer, args) .await .result { @@ -549,6 +550,13 @@ where Ok(( TypedHeader(SpacetimeIdentity(auth.claims.identity)), TypedHeader(SpacetimeIdentityToken(auth.creds)), + [ + ("x-spacetimedb-module-hash", module.info.module_hash.to_string()), + ( + "x-spacetimedb-database-identity", + module.info.database_identity.to_string(), + ), + ], response_json, )) } @@ -744,11 +752,14 @@ where let host = find_database_leader(&worker_ctx, &database).await?; let module = host.module().await.map_err(log_and_500)?; + let sql_caller_auth = caller_auth.clone(); let fut = async move |_module: ModuleHost, caller_identity: Identity, _connection_id: ConnectionId| { let sql_auth = worker_ctx .authorize_sql(caller_identity, database.database_identity) .await?; + let sql_auth = spacetimedb::auth::invocation::SqlCallAuth::authenticated(sql_auth, &sql_caller_auth) + .map_err(log_and_500)?; host.exec_sql( sql_auth, database, @@ -1082,6 +1093,11 @@ pub async fn publish( | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { tx_offset, durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. }, ) => { timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index fdb0e7fcce9..80348ea0bef 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -86,7 +86,10 @@ pub(super) fn type_ref_name(module: &ModuleDef, typeref: AlgebraicTypeRef) -> St pub(super) fn is_type_filterable(typespace: &TypespaceForGenerate, ty: &AlgebraicTypeUse) -> bool { match ty { AlgebraicTypeUse::Primitive(prim) => !matches!(prim, PrimitiveType::F32 | PrimitiveType::F64), - AlgebraicTypeUse::String | AlgebraicTypeUse::Identity | AlgebraicTypeUse::ConnectionId => true, + AlgebraicTypeUse::String + | AlgebraicTypeUse::Identity + | AlgebraicTypeUse::ConnectionId + | AlgebraicTypeUse::Uuid => true, // Sum types with all unit variants: AlgebraicTypeUse::Never => true, AlgebraicTypeUse::Option(inner) => matches!(&**inner, AlgebraicTypeUse::Unit), diff --git a/crates/codegen/tests/codegen.rs b/crates/codegen/tests/codegen.rs index b2a04309478..f4a1ff33b03 100644 --- a/crates/codegen/tests/codegen.rs +++ b/crates/codegen/tests/codegen.rs @@ -86,3 +86,45 @@ fn submodule_reducer_wire_name_is_qualified_once() { "namespace was applied twice somewhere in the generated bindings" ); } + +#[test] +fn rust_uuid_primary_and_unique_keys_generate_registered_lookup_accessors() { + use spacetimedb_lib::{ + db::raw_def::{v10::RawModuleDefV10Builder, v9::btree}, + AlgebraicType, + }; + + let mut builder = RawModuleDefV10Builder::new(); + builder + .build_table_with_new_type( + "uuid_rows", + [ + ("id", AlgebraicType::uuid()), + ("alias", AlgebraicType::uuid()), + ("sequence", AlgebraicType::U64), + ], + true, + ) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "uuid_rows_id_idx", "id") + .with_unique_constraint(1) + .with_index(btree(1), "uuid_rows_alias_idx", "alias") + .with_unique_constraint(2) + .with_index(btree(2), "uuid_rows_sequence_idx", "sequence") + .finish(); + let module = ModuleDef::try_from(builder.finish()).unwrap(); + let table = generate(&module, &Rust, &CodegenOptions::default()) + .into_iter() + .find(|file| file.filename == "uuid_rows_table.rs") + .unwrap() + .code; + for column in ["id", "alias"] { + assert!(table.contains(&format!("pub fn {column}(&self)"))); + assert!(table.contains(&format!( + "add_unique_constraint::<__sdk::Uuid>({column:?}, |row| &row.{column})" + ))); + } + assert!(table.contains("pub fn find(&self, col_val: &__sdk::Uuid) -> Option")); + assert!(table.contains("add_unique_constraint::(\"sequence\", |row| &row.sequence)")); +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0cd5fcef773..a2df624dacc 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -98,6 +98,7 @@ tracing-subscriber.workspace = true tracing-tracy.workspace = true tracing.workspace = true url.workspace = true +uuid = { workspace = true, features = ["v4", "v7"] } v8.workspace = true wasmtime.workspace = true wasmtime-internal-fiber.workspace = true diff --git a/crates/core/src/auth/hosted_tokens.rs b/crates/core/src/auth/hosted_tokens.rs new file mode 100644 index 00000000000..77622b2706a --- /dev/null +++ b/crates/core/src/auth/hosted_tokens.rs @@ -0,0 +1,332 @@ +//! Explicit platform trust for hosted database credentials. No OIDC discovery or fallback. + +use anyhow::{ensure, Context}; +use jsonwebtoken::DecodingKey; +pub use spacetimedb_auth::hosted::{ + has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, + HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, MAX_HOSTED_TOKEN_LIFETIME, +}; +use spacetimedb_auth::hosted::{unverified_hosted_token_claims, verify_hosted_token}; +use spacetimedb_lib::Identity; +use std::collections::HashMap; +use std::time::SystemTime; + +/// Only configured platform signers can attest registered source databases. +pub struct HostedTokenValidator { + trusted_issuers: HashMap, DecodingKey>, +} + +impl HostedTokenValidator { + pub fn new(issuers: impl IntoIterator, DecodingKey)>) -> anyhow::Result { + let mut trusted_issuers = HashMap::new(); + for (issuer, key) in issuers { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + trusted_issuers.insert(issuer, key).is_none(), + "duplicate trusted hosted issuer" + ); + } + Ok(Self { trusted_issuers }) + } + + /// `resolve_binding` reads authoritative state, including this issuer's source + /// registration, open admission, current placement/incarnation and target grant. + /// Return None if any requirement is absent. Its inputs are untrusted routing hints; + /// the callback must never copy them into a fabricated binding or mutate state. + /// The returned proof still requires target-fence checks at every later admission. + pub fn validate_token( + &self, + token: &str, + target: Identity, + now: SystemTime, + resolve_binding: impl FnOnce(&str, Identity, Identity) -> Option, + ) -> anyhow::Result { + let hints = unverified_hosted_token_claims(token)?; + ensure!(hints.target_database == target, "hosted credential target mismatch"); + let key = self + .trusted_issuers + .get(&hints.issuer) + .context("untrusted hosted credential issuer")?; + let binding = resolve_binding(&hints.issuer, hints.source_database, target) + .context("hosted source registration, instance, or target grant is unavailable")?; + ensure!( + binding.target_database == target, + "authoritative hosted binding target mismatch" + ); + verify_hosted_token(token, key, &hints.issuer, &binding, now) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::{ + token_validation::{FullTokenValidator, TokenValidator, UnimplementedTokenValidator}, + JwtKeys, + }; + use jsonwebtoken::{Algorithm, Header}; + use serde_json::{json, Value}; + use spacetimedb_auth::hosted::{has_reserved_hosted_token_kind, MAX_HOSTED_TOKEN_BYTES}; + use std::time::{Duration, UNIX_EPOCH}; + + fn fixture() -> (JwtKeys, HostedTokenValidator, HostedTokenBinding, SystemTime) { + let keys = JwtKeys::generate().unwrap(); + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public.clone())]).unwrap(); + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let binding = HostedTokenBinding { + source_database: Identity::from_claims("source", "database"), + target_database: Identity::from_claims("target", "database"), + generation: 9_007_199_254_740_993, + grant_revision: 9_007_199_254_740_995, + lease_expires_at: now + Duration::from_secs(30), + }; + (keys, validator, binding, now) + } + + fn mint(keys: &JwtKeys, binding: &HostedTokenBinding, now: SystemTime) -> String { + sign_hosted_token( + &keys.private, + "platform.test", + binding, + now, + now + Duration::from_secs(20), + "token-private-id", + ) + .unwrap() + } + + fn change(token: &str, keys: &JwtKeys, mutate: impl FnOnce(&mut Value, &mut Header)) -> String { + let mut claims = serde_json::to_value(unverified_hosted_token_claims(token).unwrap()).unwrap(); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + mutate(&mut claims, &mut header); + jsonwebtoken::encode(&header, &claims, &keys.private).unwrap() + } + + #[test] + fn hosted_sender_target_authority_and_claims_are_preserved() { + let (keys, validator, mut binding, now) = fixture(); + for self_call in [false, true] { + if self_call { + binding.target_database = binding.source_database; + } + let token = mint(&keys, &binding, now); + assert!(has_reserved_hosted_token_kind(&token).unwrap()); + let verified = validator + .validate_token(&token, binding.target_database, now, |issuer, source, target| { + assert_eq!(issuer, "platform.test"); + assert_eq!(source, binding.source_database); + assert_eq!(target, binding.target_database); + Some(binding) + }) + .unwrap(); + assert_eq!(verified.is_internal(), self_call); + assert_eq!(verified.generation(), binding.generation); + assert_eq!(verified.grant_revision(), binding.grant_revision); + assert!(verified.check_at(now + Duration::from_secs(20)).is_err()); + let ctx = verified.into_connection_auth().unwrap(); + assert_eq!(ctx.claims.identity, binding.source_database); + assert_ne!( + ctx.claims.identity, + Identity::from_claims(&ctx.claims.issuer, &ctx.claims.subject) + ); + assert!(ctx.hosted.is_some()); + let payload: Value = serde_json::from_str(&ctx.jwt_payload).unwrap(); + assert_eq!(payload["generation"].as_u64(), Some(binding.generation)); + assert_eq!(payload["grant_revision"].as_u64(), Some(binding.grant_revision)); + assert_eq!(payload["aud"], binding.target_database.to_hex().as_str()); + assert_eq!(payload["iss"], "platform.test"); + let debug = format!("{ctx:?}"); + assert!(!debug.contains("token-private-id")); + assert!(!debug.contains(&token)); + assert!(!debug.contains("jwt_payload")); + } + } + + #[test] + fn hosted_validation_rejects_wrong_authority_binding_and_wire_shape() { + let (keys, validator, binding, now) = fixture(); + let token = mint(&keys, &binding, now); + let other_keys = JwtKeys::generate().unwrap(); + assert!(validator + .validate_token( + &mint(&other_keys, &binding, now), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + assert!(validator + .validate_token(&token, binding.source_database, now, |_, _, _| Some(binding)) + .is_err()); + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| None) + .is_err()); + let invalid_fields = [ + ("kind", json!("spacetimedb_hosted_v2")), + ("iss", json!("unknown.test")), + ("source_database", json!(binding.target_database.to_hex().as_str())), + ("sub", json!("other")), + ("aud", json!(binding.source_database.to_hex().as_str())), + ("aud", json!([binding.target_database.to_hex().as_str()])), + ("generation", json!(binding.generation - 1)), + ("grant_revision", json!(binding.grant_revision - 1)), + ("iat", json!(1_700_000_001_u64)), + ("exp", json!(1_700_000_000_u64)), + ("exp", json!(1_700_000_031_u64)), + ("exp", json!(u64::MAX)), + ("jti", json!("")), + ("hex_identity", json!(binding.source_database.to_hex().as_str())), + ]; + for (field, value) in invalid_fields { + let changed = change(&token, &keys, |claims, _| claims[field] = value); + assert!( + validator + .validate_token(&changed, binding.target_database, now, |_, _, _| Some(binding)) + .is_err(), + "accepted changed {field}" + ); + } + let wrong_type = change(&token, &keys, |_, header| header.typ = Some("JWT".into())); + assert!(validator + .validate_token(&wrong_type, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let missing_exp = change(&token, &keys, |claims, _| { + claims.as_object_mut().unwrap().remove("exp"); + }); + assert!(validator + .validate_token(&missing_exp, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let overflowing_time = change(&token, &keys, |claims, _| { + claims["iat"] = json!(u64::MAX - 20); + claims["exp"] = json!(u64::MAX); + }); + assert!(validator + .validate_token(&overflowing_time, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(19), + ..binding + }; + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| Some(short_lease)) + .is_err()); + assert!(validator + .validate_token( + &"x".repeat(MAX_HOSTED_TOKEN_BYTES + 1), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + let claims = unverified_hosted_token_claims(&token).unwrap(); + let mut hs_header = Header::new(Algorithm::HS256); + hs_header.typ = Some(HOSTED_TOKEN_TYPE.into()); + let hs_token = jsonwebtoken::encode( + &hs_header, + &claims, + &jsonwebtoken::EncodingKey::from_secret(b"not-a-platform-key"), + ) + .unwrap(); + assert!(validator + .validate_token(&hs_token, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + } + + #[test] + fn broker_signing_obeys_confirmed_lease_and_lifetime() { + let (keys, _, binding, now) = fixture(); + for expiry in [now, now + Duration::from_secs(31)] { + assert!(sign_hosted_token(&keys.private, "platform.test", &binding, now, expiry, "id").is_err()); + } + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(10), + ..binding + }; + assert!(sign_hosted_token( + &keys.private, + "platform.test", + &short_lease, + now, + now + Duration::from_secs(11), + "id" + ) + .is_err()); + } + + #[tokio::test] + async fn ordinary_validation_rejects_reserved_platform_kinds_and_types() { + let (keys, _, binding, _) = fixture(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(30), + ..binding + }; + let token = mint(&keys, &binding, now); + let ordinary = FullTokenValidator { + local_key: keys.public.clone(), + local_issuer: "platform.test".into(), + oidc_validator: UnimplementedTokenValidator, + }; + for reserved in [ + token.clone(), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_hosted_future"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-hosted-v2+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_lease_v1"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-container-lease+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_registry_future"); + header.typ = Some("JWT".into()); + }), + ] { + assert!(spacetimedb_auth::hosted::has_reserved_platform_token_kind(&reserved).unwrap()); + assert!(keys.public.validate_token(&reserved).await.is_err()); + assert!(ordinary.validate_token(&reserved).await.is_err()); + } + } + + #[tokio::test] + async fn reserved_classification_preserves_ordinary_token_algorithms() { + let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = openssl::pkey::PKey::from_rsa(rsa).unwrap(); + let ec = JwtKeys::generate().unwrap(); + let keys = [ + (Algorithm::ES256, ec.private, ec.public), + ( + Algorithm::RS256, + jsonwebtoken::EncodingKey::from_rsa_pem(&rsa.private_key_to_pem_pkcs8().unwrap()).unwrap(), + DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(), + ), + ( + Algorithm::HS256, + jsonwebtoken::EncodingKey::from_secret(b"ordinary-oidc-test-secret"), + DecodingKey::from_secret(b"ordinary-oidc-test-secret"), + ), + ]; + for (algorithm, private, public) in keys { + let claims = json!({ "iss": "ordinary.test", "sub": "a-user", "iat": 1_700_000_000_u64, "kind": "ordinary_application_kind" }); + let token = jsonwebtoken::encode(&Header::new(algorithm), &claims, &private).unwrap(); + assert!( + !has_reserved_hosted_token_kind(&token).unwrap(), + "misclassified {algorithm:?}" + ); + let validated = public.validate_token(&token).await.unwrap(); + assert_eq!(validated.identity, Identity::from_claims("ordinary.test", "a-user")); + } + } +} diff --git a/crates/core/src/auth/invocation.rs b/crates/core/src/auth/invocation.rs new file mode 100644 index 00000000000..dc9004effaa --- /dev/null +++ b/crates/core/src/auth/invocation.rs @@ -0,0 +1,127 @@ +//! Authority carried from authenticated admission to module execution. +//! +//! Identity equality never establishes internal authority. A hosted proof can +//! only originate in signature verification against trusted platform state. + +use super::hosted_tokens::VerifiedHostedAuth; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::Identity; +use spacetimedb_schema::def::ModuleDef; +use std::time::SystemTime; + +#[derive(Clone, Debug)] +pub struct InvocationCaller { + pub(crate) identity: Identity, + pub(crate) hosted: Option>, +} + +impl From for InvocationCaller { + fn from(identity: Identity) -> Self { + Self { identity, hosted: None } + } +} + +impl From<&ConnectionAuthCtx> for InvocationCaller { + fn from(auth: &ConnectionAuthCtx) -> Self { + Self { + identity: auth.claims.identity, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + } + } +} + +/// SQL permissions and the authenticated container restrictions are independent. +/// Internal status never impersonates the database owner or grants SQL rights. +#[derive(Clone)] +pub struct SqlCallAuth { + permissions: spacetimedb_lib::identity::AuthCtx, + pub(crate) hosted: Option>, +} + +impl From for SqlCallAuth { + fn from(permissions: spacetimedb_lib::identity::AuthCtx) -> Self { + Self { + permissions, + hosted: None, + } + } +} + +impl std::ops::Deref for SqlCallAuth { + type Target = spacetimedb_lib::identity::AuthCtx; + fn deref(&self) -> &Self::Target { + &self.permissions + } +} + +impl SqlCallAuth { + pub fn authenticated( + permissions: spacetimedb_lib::identity::AuthCtx, + auth: &ConnectionAuthCtx, + ) -> anyhow::Result { + anyhow::ensure!( + permissions.caller() == auth.claims.identity, + "SQL caller does not match authenticated sender" + ); + if let Some(proof) = &auth.hosted { + anyhow::ensure!( + proof.source_database() == auth.claims.identity, + "SQL hosted proof does not match authenticated sender" + ); + } + Ok(Self { + permissions, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + }) + } +} + +impl InvocationCaller { + pub(crate) fn flags_for(&self, target: Identity, module: &ModuleDef) -> anyhow::Result { + let Some(proof) = &self.hosted else { return Ok(0) }; + anyhow::ensure!( + proof.source_database() == self.identity, + "hosted caller does not match its proof" + ); + anyhow::ensure!( + proof.target_database() == target, + "hosted credential targets another database" + ); + anyhow::ensure!( + module.supports_hosted_auth_v1(), + "module does not support hosted authentication" + ); + proof.check_at(SystemTime::now())?; + Ok(u32::from(proof.is_internal())) + } +} + +/// Must run while holding the transaction that admits the database operation. +/// A check before queueing does not serialize with generation revocation. +pub(crate) fn check_hosted_admission( + state: &S, + database: &crate::db::relational_db::RelationalDB, + proof: Option<&VerifiedHostedAuth>, +) -> anyhow::Result<()> { + let Some(proof) = proof else { return Ok(()) }; + anyhow::ensure!( + database.hosted_admission().is_open(), + "receiving database has not reconciled hosted admission" + ); + anyhow::ensure!( + proof.target_database() == database.database_identity(), + "hosted credential targets another database" + ); + proof.check_at(SystemTime::now())?; + crate::db::deployment::check_container_fence( + state, + proof.source_database(), + proof.generation(), + proof.grant_revision(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs new file mode 100644 index 00000000000..0507ca32bdb --- /dev/null +++ b/crates/core/src/auth/invocation/tests.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use crate::auth::JwtKeys; +use crate::db::deployment::install_container_fence; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_lib::db::auth::StAccess; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_lib::identity::AuthCtx; +use std::time::Duration; + +fn module(hosted_auth: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + if hosted_auth { + builder.add_capability("hosted_auth_v1"); + } + builder.finish().try_into().unwrap() +} + +/// Obtain every proof through the production signer and target-bound verifier. +fn authenticate(source: Identity, target: Identity, now: SystemTime) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 3, + grant_revision: 7, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + Duration::from_secs(20), + "invocation-test", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn fence(source: Identity, generation: u64, grant_revision: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: grant_revision, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + } +} + +#[test] +fn internal_requires_verified_self_call_and_updated_bindings() { + let target = Identity::ONE; + let foreign = Identity::from_u256(2u64.into()); + let updated = module(true); + let old_bindings = module(false); + // This also covers an ordinary connection whose sender equals the database. + assert_eq!(InvocationCaller::from(target).flags_for(target, &updated).unwrap(), 0); + for (source, expected_flags) in [(target, 1), (foreign, 0)] { + let auth = authenticate(source, target, SystemTime::now()); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &updated).unwrap(), expected_flags); + assert!(caller.flags_for(target, &old_bindings).is_err()); + assert!(caller.flags_for(Identity::ZERO, &updated).is_err()); + } +} + +#[test] +fn authenticated_proof_cannot_be_paired_with_another_sender_or_sql_caller() { + let source = Identity::ONE; + let target = Identity::from_u256(2u64.into()); + let owner = Identity::from_u256(3u64.into()); + let mut auth = authenticate(source, target, SystemTime::now()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); + // ConnectionAuthCtx has public fields for trusted host code. Defend the + // boundary against accidentally mixing separately authenticated contexts. + auth.claims.identity = owner; + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); +} + +#[test] +fn internal_authentication_does_not_grant_owner_sql_permissions() { + let source = Identity::ONE; + let owner = Identity::from_u256(3u64.into()); + let auth = authenticate(source, source, SystemTime::now()); + assert_eq!( + InvocationCaller::from(&auth).flags_for(source, &module(true)).unwrap(), + 1 + ); + let sql = SqlCallAuth::authenticated(AuthCtx::new(owner, source), &auth).unwrap(); + assert_eq!(sql.caller(), source); + assert!(sql.has_read_access(StAccess::Public)); + assert!(!sql.has_read_access(StAccess::Private)); + assert!(!sql.has_write_access()); + assert!(!sql.bypass_rls()); +} + +#[test] +fn transaction_admission_rechecks_persisted_fences_after_initial_authentication() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + let auth = authenticate(source, target, SystemTime::now()); + let proof = auth.hosted.as_ref(); + let wrong_target = authenticate(source, Identity::from_u256(99u64.into()), SystemTime::now()); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &module(true)).unwrap(), 0); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(!db.hosted_admission().is_open()); + db.hosted_admission().begin()?.complete()?; + check_hosted_admission(tx, &db, proof)?; + assert!(check_hosted_admission(tx, &db, wrong_target.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); + // Revocation commits after token verification and before the queued call. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 4, 8, false))?; + Ok(()) + }) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + check_hosted_admission(tx, &db, None)?; + // Another generation does not reactivate a copied credential. + install_container_fence(&db, tx, &fence(source, 5, 9, true))?; + assert!(check_hosted_admission(tx, &db, proof).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn reopened_database_requires_a_fresh_sweep_despite_a_replayed_allowed_fence() { + let db = TestDB::durable().unwrap(); + let source = Identity::ONE; + let target = db.database_identity(); + let auth = authenticate(source, target, SystemTime::now()); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(source, 3, 7, true)) + }) + .unwrap(); + let old_sweep = db.hosted_admission().begin().unwrap(); + let db = db.reopen().unwrap(); + // Shutdown seals the old object, including any late coordinator ticket. + assert!(old_sweep.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + db.with_read_only(Workload::ForTests, |tx| { + crate::db::deployment::check_container_fence(tx, source, 3, 7).unwrap(); + let error = check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap_err(); + assert!(error.to_string().contains("has not reconciled")); + check_hosted_admission(tx, &db, None).unwrap(); + }); + db.hosted_admission().begin().unwrap().complete().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap(); + }); +} + +#[test] +fn expired_verified_proof_is_rejected_at_both_call_and_transaction_admission() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + db.hosted_admission().begin().unwrap().complete().unwrap(); + // Valid when received, expired before execution, without sleeps or forged proofs. + let auth = authenticate(source, target, SystemTime::now() - Duration::from_secs(60)); + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(check_hosted_admission(tx, &db, auth.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn shutdown_seals_hosted_admission_even_for_retained_memory_database_handles() { + let db = TestDB::in_memory().unwrap(); + let pending = db.hosted_admission().begin().unwrap(); + assert_eq!(db.shutdown().await, None); + assert!(pending.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + assert!(db.hosted_admission().begin().is_err()); + assert_eq!(db.shutdown().await, None); +} diff --git a/crates/core/src/auth/mod.rs b/crates/core/src/auth/mod.rs index e1e38a667c4..49f2686f062 100644 --- a/crates/core/src/auth/mod.rs +++ b/crates/core/src/auth/mod.rs @@ -7,6 +7,8 @@ use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use crate::config::CertificateAuthority; pub use spacetimedb_auth::identity; +pub mod hosted_tokens; +pub mod invocation; pub mod token_validation; /// JWT verification and signing keys. diff --git a/crates/core/src/auth/token_validation.rs b/crates/core/src/auth/token_validation.rs index 22ebcef743e..0aacfeb0839 100644 --- a/crates/core/src/auth/token_validation.rs +++ b/crates/core/src/auth/token_validation.rs @@ -96,6 +96,7 @@ where T: TokenValidator + Send + Sync, { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let local_key_error = { let first_validator = BasicTokenValidator { public_key: self.local_key.clone(), @@ -144,6 +145,7 @@ lazy_static! { #[async_trait] impl TokenValidator for DecodingKey { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.algorithms = match self.family() { AlgorithmFamily::Ec => vec![jsonwebtoken::Algorithm::ES256], @@ -269,6 +271,7 @@ pub struct OidcTokenValidator; // Get the issuer out of a token without validating the signature. fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { + reject_reserved_hosted_credentials(token)?; // We need the issuer before we know which key to use. // This intentionally does not validate the token. // Callers must only use it for key discovery and must verify the token afterwards. @@ -276,6 +279,16 @@ fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { Ok(data.claims.issuer) } +fn reject_reserved_hosted_credentials(token: &str) -> Result<(), TokenValidationError> { + if spacetimedb_auth::hosted::has_reserved_platform_token_kind(token)? { + return Err(anyhow::anyhow!( + "platform container credentials require their dedicated validator and cannot be exchanged" + ) + .into()); + } + Ok(()) +} + #[async_trait] impl TokenValidator for OidcTokenValidator { async fn validate_token(&self, token: &str) -> Result { diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 23e801e4b30..3aa66d91572 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -8,6 +8,7 @@ use std::task::{Context, Poll}; use std::time::{Instant, SystemTime}; use super::{message_handlers, ClientActorId, MessageHandleError, OutboundMessage}; +use crate::auth::{hosted_tokens::VerifiedHostedAuth, invocation::check_hosted_admission}; use crate::db::relational_db::RelationalDB; use crate::error::DBError; use crate::host::module_host::{ClientConnectedError, ProcedureResultTarget}; @@ -25,6 +26,7 @@ use prometheus::{Histogram, IntCounter, IntGauge}; use scopeguard::ScopeGuard; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; +use spacetimedb_datastore::execution_context::Workload; use spacetimedb_durability::{DurableOffset, TxOffset}; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; @@ -128,9 +130,29 @@ pub trait DurableOffsetSupply: Send { /// - `Ok(Some(DurableOffset))` otherwise /// fn durable_offset(&mut self) -> Result, NoSuchModule>; + + /// Recheck the authoritative target state, never a cached generation. + fn check_hosted_auth( + &mut self, + _proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { anyhow::bail!("hosted connection has no authoritative database state") }) + } } impl DurableOffsetSupply for watch::Receiver { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + if self.has_changed().is_err() { + return Box::pin(async { Err(NoSuchModule.into()) }); + } + let module = self.borrow().clone(); + let mut db = module.relational_db().clone(); + db.check_hosted_auth(proof) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { let module = if self.has_changed().map_err(|_| NoSuchModule)? { self.borrow_and_update() @@ -143,6 +165,20 @@ impl DurableOffsetSupply for watch::Receiver { } impl DurableOffsetSupply for Arc { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + let db = self.clone(); + let proof = proof.clone(); + Box::pin(async move { + tokio::task::spawn_blocking(move || { + db.with_read_only(Workload::Internal, |tx| check_hosted_admission(tx, &db, Some(&proof))) + }) + .await? + }) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { Ok(self.durable_tx_offset()) } @@ -160,6 +196,7 @@ pub struct ClientConnectionReceiver { channel: MeteredReceiver, pending: Vec, offset_supply: Box, + hosted_sender: Option>, } impl ClientConnectionReceiver { @@ -175,6 +212,7 @@ impl ClientConnectionReceiver { channel, pending: Vec::new(), offset_supply: Box::new(offset_supply), + hosted_sender: None, } } @@ -217,6 +255,9 @@ impl ClientConnectionReceiver { /// These values are stored internally, so calling `recv_many` again will /// not lose data. pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + if !self.hosted_connection_is_valid().await { + return 0; + } // If there are no pending updates and the input channel has been closed, // no more messages can be received from this receiver. if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { @@ -226,14 +267,14 @@ impl ClientConnectionReceiver { // If we don't have to wait for txns to be made durable, // drain the pending updates. if !self.confirmed_reads { - return self.drain_pending(buf, max); + return self.drain_pending(buf, max).await; } // If we do have to wait for txns to be made durable, // but the next client update doesn't have a tx offset, // there's no reason to wait - just send it. if !self.pending_update_has_offset() { - return self.drain_pending(buf, 1); + return self.drain_pending(buf, 1).await; } // Otherwise, grab the next offset that we should wait for. @@ -246,12 +287,12 @@ impl ClientConnectionReceiver { warn!("database went away while waiting for durable offset"); return 0; } - self.drain_pending(buf, n) + self.drain_pending(buf, n).await } // Database shut down or crashed. Err(NoSuchModule) => 0, // In-memory database. - Ok(None) => self.drain_pending(buf, max), + Ok(None) => self.drain_pending(buf, max).await, } } @@ -268,13 +309,40 @@ impl ClientConnectionReceiver { } /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. - fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + async fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + // A queued update may predate revocation, and a confirmed-read wait may + // outlast the credential. Check again immediately before delivery. + if !self.hosted_connection_is_valid().await { + return 0; + } let n = self.pending.len().min(max); buf.reserve(n); buf.extend(self.pending.drain(..n).map(|u| u.message)); n } + async fn hosted_connection_is_valid(&mut self) -> bool { + let Some(sender) = &self.hosted_sender else { return true }; + let valid = match sender.upgrade() { + Some(sender) => { + let valid = match &sender.auth.hosted { + Some(proof) if !sender.is_cancelled() => self.offset_supply.check_hosted_auth(proof).await.is_ok(), + _ => false, + }; + if !valid { + sender.cancel_hosted_connection(); + } + valid + } + None => false, + }; + if !valid { + self.pending.clear(); + self.close(); + } + valid + } + /// Does the next pending update have a tx offset? /// /// Assumes that [`Self::pending`] is not empty. @@ -365,6 +433,13 @@ pub enum ClientSendError { } impl ClientConnectionSender { + /// The fence installer awaits the returned task's completion before ack. + pub(crate) fn cancel_hosted_connection(&self) -> AbortHandle { + self.cancelled.store(true, Ordering::Release); + self.abort_handle.abort(); + self.abort_handle.clone() + } + pub fn dummy_with_channel( id: ClientActorId, config: ClientConfig, @@ -434,6 +509,16 @@ impl ClientConnectionSender { } fn send(&self, message: ClientUpdate) -> Result<(), ClientSendError> { + // Do not acquire a database transaction here: broadcasts can already + // hold one. Durable fencing is checked at admission and delivery. + if self + .auth + .hosted + .as_ref() + .is_some_and(|proof| proof.check_at(SystemTime::now()).is_err()) + { + self.cancel_hosted_connection(); + } if self.cancelled.load(Relaxed) { return Err(ClientSendError::Cancelled); } @@ -486,6 +571,65 @@ impl ClientConnectionSender { } } +/// Runs independently of the socket actor, so blocked writes and idle sockets +/// cannot keep credentials alive. Expiry uses a monotonic deadline captured once. +fn spawn_hosted_connection_watchdog( + sender: std::sync::Weak, + mut supply: impl DurableOffsetSupply + 'static, + subscriptions: Option, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let actor = sender.upgrade().map(|connection| connection.abort_handle.clone()); + async { + let Some(connection) = sender.upgrade() else { return }; + let Some(proof) = connection.auth.hosted.clone() else { + return; + }; + let deadline = tokio::time::Instant::now() + proof.remaining_lifetime(SystemTime::now()); + drop(connection); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + if let Some(connection) = sender.upgrade() { connection.cancel_hosted_connection(); } + return; + } + _ = interval.tick() => {} + } + let Some(connection) = sender.upgrade() else { return }; + if connection.abort_handle.is_finished() || connection.is_cancelled() { + return; + } + let checked = tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + connection.cancel_hosted_connection(); + return; + } + checked = supply.check_hosted_auth(&proof) => checked, + }; + if checked.is_err() { + connection.cancel_hosted_connection(); + return; + } + } + } + .await; + // Keep the registry entry until socket I/O has actually stopped. A + // concurrent target barrier must still find and await an aborted actor. + if let Some(actor) = actor { + while !actor.is_finished() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + if let Some(subscriptions) = subscriptions { + subscriptions.unregister_hosted_connection(&sender); + } + }) +} + #[derive(Clone)] #[non_exhaustive] pub struct ClientConnection { @@ -900,7 +1044,7 @@ impl ClientConnection { }) .abort_handle(); - let receiver = ClientConnectionReceiver::new( + let mut receiver = ClientConnectionReceiver::new( config.confirmed_reads, MeteredReceiver::with_gauge(sendrx, metrics.sendtx_queue_size.clone()), module_rx.clone(), @@ -915,6 +1059,18 @@ impl ClientConnection { cancelled: AtomicBool::new(false), metrics: Some(metrics), }); + if sender.auth.hosted.is_some() { + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + if module.subscriptions().register_hosted_connection(&sender).is_err() { + sender.cancel_hosted_connection(); + } else { + spawn_hosted_connection_watchdog( + Arc::downgrade(&sender), + module_rx.clone(), + Some(module.subscriptions().clone()), + ); + } + } let this = Self { sender, replica_id, @@ -1031,7 +1187,7 @@ impl ClientConnection { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1052,7 +1208,7 @@ impl ClientConnection { ) -> Result { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1078,7 +1234,7 @@ impl ClientConnection { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1099,7 +1255,7 @@ impl ClientConnection { ) -> Result<(), ReducerCallError> { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1119,7 +1275,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1139,7 +1295,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1368,6 +1524,354 @@ mod tests { assert_matches!(futures::poll!(f), Poll::Pending); } + fn hosted_auth(db: &RelationalDB, lifetime: std::time::Duration) -> ConnectionAuthCtx { + // These fixtures model an already reconciled receiving host. Tests of + // startup closure explicitly close the gate after constructing it. + if !db.hosted_admission().is_open() { + db.hosted_admission().begin().unwrap().complete().unwrap(); + } + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: now + std::time::Duration::from_secs(30), + }; + let token = sign_hosted_token(&keys.private, "platform.test", &binding, now, now + lifetime, "test").unwrap(); + HostedTokenValidator::new([("platform.test".into(), keys.public)]) + .unwrap() + .validate_token(&token, db.database_identity(), now, |_, _, _| Some(binding)) + .unwrap() + .into_connection_auth() + .unwrap() + } + + fn set_fence(db: &RelationalDB, generation: u64, allowed: bool) { + use crate::db::deployment::install_container_fence; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed, + }, + ) + }) + .unwrap(); + } + + fn hosted_client( + db: &RelationalDB, + supply: impl DurableOffsetSupply + 'static, + confirmed_reads: bool, + lifetime: std::time::Duration, + ) -> ( + Arc, + ClientConnectionReceiver, + tokio::task::JoinHandle<()>, + ) { + let (mut sender, mut receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig { + confirmed_reads, + ..ClientConfig::for_test() + }, + supply, + ); + sender.auth = hosted_auth(db, lifetime); + let actor = tokio::spawn(std::future::pending()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + (sender, receiver, actor) + } + + struct HostedConfirmedSupply { + db: Arc, + durable: FakeDurableOffset, + durability_requested: Arc, + } + impl DurableOffsetSupply for HostedConfirmedSupply { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + self.durability_requested.store(true, Ordering::Release); + self.durable.durable_offset() + } + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + self.db.check_hosted_auth(proof) + } + } + + #[tokio::test] + async fn hosted_queued_delivery_rechecks_committed_fence_and_preserves_ordinary_identity() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + set_fence(&db, 2, false); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_queued_delivery_rejects_closed_startup_gate_with_unchanged_fence() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + db.hosted_admission().close(); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_confirmed_delivery_rechecks_fence_after_durability_wait() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let durable = FakeDurableOffset::new(); + let durability_requested = Arc::new(AtomicBool::new(false)); + let supply = HostedConfirmedSupply { + db: db.db.clone(), + durable: durable.clone(), + durability_requested: durability_requested.clone(), + }; + let (sender, mut receiver, actor) = hosted_client(&db, supply, true, std::time::Duration::from_secs(20)); + sender.send_message(Some(7), empty_tx_update()).unwrap(); + let mut receiving = Box::pin(receiver.recv()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !durability_requested.load(Ordering::Acquire) { + assert_pending(&mut receiving).await; + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + set_fence(&db, 2, false); + durable.mark_durable_at(7); + assert_receiver_closed(receiving).await; + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_expires_without_outbound_traffic() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(2)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + tokio::time::timeout(std::time::Duration::from_secs(3), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_connection_uses_confirmed_expiry_while_authority_read_is_blocked() { + struct BlockedAuthority; + impl DurableOffsetSupply for BlockedAuthority { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + Ok(None) + } + + fn check_hosted_auth( + &mut self, + _: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(std::future::pending()) + } + } + + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (mut sender, _receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig::for_test(), + db.db.clone(), + ); + let proof = hosted_auth(&db, std::time::Duration::from_secs(20)).hosted.unwrap(); + let confirmed_time = proof.expires_at() - std::time::Duration::from_secs(5); + // Authority is ahead of the receiving wall clock. Almost all of the + // five remaining seconds elapsed while its confirmation was in flight. + let started = std::time::Instant::now() - std::time::Duration::from_millis(4_900); + sender.auth = proof + .constrain_expiration(confirmed_time, started) + .unwrap() + .into_connection_auth() + .unwrap(); + let actor = tokio::spawn(std::future::pending::<()>()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), BlockedAuthority, None); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_rechecks_durable_revocation() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_watchdog_ends_and_unregisters_after_socket_actor_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + actor.abort(); + let _ = actor.await; + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 0); + // The registry releases the entry even while another owner retains sender. + assert!(!sender.is_cancelled()); + } + + #[tokio::test] + async fn hosted_registry_retains_cancelled_actor_until_delivery_cleanup_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (mut sender, _receiver) = default_client(db.db.clone()); + sender.auth = hosted_auth(&db, std::time::Duration::from_secs(20)); + let (release, blocked) = std::sync::mpsc::channel(); + let (started, ready) = oneshot::channel(); + // A started blocking task models cleanup that cannot complete merely + // because abort was requested. The barrier must retain its handle. + let actor = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + }); + ready.await.unwrap(); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + subscriptions.register_hosted_connection(&sender).unwrap(); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !sender.is_cancelled() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let handles = db.with_read_only(Workload::ForTests, |tx| { + subscriptions.cancel_invalid_hosted_connections(tx) + }); + assert_eq!(handles.len(), 1); + assert!(!handles[0].is_finished()); + release.send(()).unwrap(); + actor.await.unwrap(); + watchdog.await.unwrap(); + assert!(handles[0].is_finished()); + assert_eq!(subscriptions.hosted_connection_count(), 0); + } + + #[tokio::test] + async fn hosted_target_barrier_cancels_connections_without_subscriptions_and_waits_for_actor() { + use crate::db::deployment::install_container_fence; + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + let handles = db + .with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + &db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation: 2, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed: true, + }, + )?; + Ok::<_, anyhow::Error>(subscriptions.cancel_invalid_hosted_connections(tx)) + }) + .unwrap(); + assert_eq!(handles.len(), 1); + assert!(actor.await.unwrap_err().is_cancelled()); + assert!(handles[0].is_finished()); + assert!(subscriptions.register_hosted_connection(&sender).is_err()); + } + + #[tokio::test] + async fn hosted_subscription_rejected_under_transaction_before_query_compilation() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + // There is deliberately no installed fence. The malformed SQL verifies + // authentication fails before query parsing or view materialization. + let result = subscriptions + .add_legacy_subscriber( + None, + sender.clone(), + AuthCtx::new(db.database_identity(), db.database_identity()), + ws_v1::Subscribe { + query_strings: ["invalid SQL".into()].into(), + request_id: 0, + }, + Instant::now(), + None, + ) + .await; + assert!(result.unwrap_err().to_string().contains("container")); + sender.cancel_hosted_connection(); + let _ = actor.await; + } + fn default_client( offset_supply: impl DurableOffsetSupply + 'static, ) -> (ClientConnectionSender, ClientConnectionReceiver) { diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 8dc9ceb7273..9e5b314fcfe 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -175,6 +175,7 @@ impl<'de> serde::Deserialize<'de> for ConfigFile { v8: V8Config { procedure_instance_pool_size: config.v8.procedure_instance_pool_size, heap_policy: config.v8_heap_policy, + execution_timeout: config.v8.execution_timeout, }, }) } @@ -265,6 +266,9 @@ impl Default for WasmConfigToml { pub struct V8Config { pub procedure_instance_pool_size: NonZeroUsize, pub heap_policy: V8HeapPolicyConfig, + /// Wall-clock limit for one JavaScript startup, description or function call. + /// Must be positive and no greater than 120 seconds. + pub execution_timeout: Duration, } impl Default for V8Config { @@ -272,11 +276,20 @@ impl Default for V8Config { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), heap_policy: V8HeapPolicyConfig::default(), + execution_timeout: default_v8_execution_timeout(), } } } impl V8Config { + pub fn validate_execution_timeout(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.execution_timeout.is_zero() && self.execution_timeout <= default_v8_execution_timeout(), + "V8 execution timeout must be positive and no greater than 120 seconds" + ); + Ok(()) + } + pub fn normalized(mut self) -> Self { self.heap_policy = self.heap_policy.normalized(); self @@ -291,16 +304,35 @@ struct V8ConfigToml { deserialize_with = "de_nz_usize" )] pub procedure_instance_pool_size: NonZeroUsize, + #[serde( + default = "default_v8_execution_timeout", + deserialize_with = "de_v8_execution_timeout" + )] + pub execution_timeout: Duration, } impl Default for V8ConfigToml { fn default() -> Self { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), + execution_timeout: default_v8_execution_timeout(), } } } +fn default_v8_execution_timeout() -> Duration { + Duration::from_secs(120) +} + +fn de_v8_execution_timeout<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let timeout = de_nz_duration(deserializer)? + .filter(|timeout| *timeout <= default_v8_execution_timeout()) + .ok_or_else(|| { + serde::de::Error::custom("V8 execution timeout must be positive and no greater than 120 seconds") + })?; + Ok(timeout) +} + #[derive(Clone, Copy, Debug, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub struct V8HeapPolicyConfig { @@ -576,6 +608,25 @@ mod tests { .unwrap_err(); } + #[test] + fn v8_execution_timeout_is_finite_and_bounded() { + let defaults: ConfigFile = toml::from_str("").unwrap(); + assert_eq!(defaults.v8.execution_timeout, Duration::from_secs(120)); + for value in ["0", "121", "\"0s\"", "\"121s\""] { + assert!(toml::from_str::(&format!("[v8]\nexecution-timeout = {value}")).is_err()); + } + let config: ConfigFile = toml::from_str("[v8]\nexecution-timeout = \"150ms\"").unwrap(); + assert_eq!(config.v8.execution_timeout, Duration::from_millis(150)); + for timeout in [Duration::ZERO, Duration::from_secs(121)] { + assert!(V8Config { + execution_timeout: timeout, + ..V8Config::default() + } + .validate_execution_timeout() + .is_err()); + } + } + #[test] fn v8_heap_policy_defaults_when_omitted() { let config: ConfigFile = toml::from_str("").unwrap(); diff --git a/crates/core/src/db/container_environment.rs b/crates/core/src/db/container_environment.rs new file mode 100644 index 00000000000..fcf5c0f1fd0 --- /dev/null +++ b/crates/core/src/db/container_environment.rs @@ -0,0 +1,362 @@ +//! Transactional immutable container environment snapshots. +//! +//! These are host-only operations. Before invoking them, the adapter must +//! authenticate the assigned service and confirm the exact current control +//! intent and authoritative leader. Identity equality and a cached control row +//! are insufficient. Helpers retain the caller's transaction and do no IO. +//! +//! Historical restore must close admission and reconcile newer operational +//! fences before any snapshot request. A missing Ready snapshot is an error, +//! never permission to recapture values from a restored or current `st_env`. + +use super::{ + deployment, environment, + relational_db::{MutTx, RelationalDB}, +}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerEnvironmentRow, StContainerFenceRow, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, +}; +use spacetimedb_lib::container::{validate_env_key, validate_exec_size, ContainerSpec, MAX_ENV_KEYS}; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use spacetimedb_lib::{bsatn, SpacetimeType, Uuid}; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::{collections::BTreeMap, fmt}; + +pub const MAX_SNAPSHOT_BYTES: usize = 256 * 1024; +pub const MAX_RETAINED_SNAPSHOTS: u64 = 64; + +/// Error text and Debug never include selected values or serialized records. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum EnvironmentSnapshotError { + #[error("container environment request is invalid")] + InvalidScope, + #[error("container environment generation is fenced")] + Fenced, + #[error("container environment committed publication does not match")] + RevisionConflict, + #[error("container environment snapshot belongs to another immutable scope")] + ScopeConflict, + #[error("container environment snapshot has not been captured")] + NotCaptured, + #[error("required container environment keys are missing: {0:?}")] + MissingKeys(Vec), + #[error("container environment does not satisfy startup limits")] + InvalidEnvironment, + #[error("container environment snapshot capacity exhausted")] + Capacity, + #[error("container environment snapshot metadata is invalid")] + CorruptMetadata, + #[error("container environment storage operation failed")] + Storage, + #[error("container environment requires durable storage")] + DurabilityUnavailable, + #[error("container environment durability could not be confirmed")] + DurabilityFailed, +} + +impl From for EnvironmentSnapshotError { + fn from(_: crate::error::DBError) -> Self { + Self::Storage + } +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum Record { + V1(RecordV1), +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct RecordV1 { + receipt: EnvironmentSnapshotReceipt, + selected_values: Vec, +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CapturedValue { + key: String, + value: String, +} + +/// Selected database values only. The trusted adapter merges verified image +/// defaults and current platform variables, then validates the complete exec. +pub struct SecretEnvironment { + pub receipt: EnvironmentSnapshotReceipt, + pub selected_values: BTreeMap, +} + +impl fmt::Debug for SecretEnvironment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretEnvironment") + .field("receipt", &self.receipt) + .field("selected_values", &"[redacted]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnvironmentClosedReceipt { + pub scope: EnvironmentSnapshotScope, + pub closed_through_generation: u64, +} + +fn valid_uuid(id: Uuid) -> bool { + matches!( + id.get_version(), + Some(spacetimedb_sats::uuid::Version::V4 | spacetimedb_sats::uuid::Version::V7) + ) +} + +fn validate_scope(db: &RelationalDB, scope: &EnvironmentSnapshotScope) -> Result<(), EnvironmentSnapshotError> { + if db.database_identity() != scope.database_identity + || scope.database_id == 0 + || scope.node_id == 0 + || scope.generation == 0 + || scope.publication_epoch == 0 + || scope.publication_operation.get_version() != Some(spacetimedb_sats::uuid::Version::V7) + || scope.cluster.is_empty() + || scope.cluster.len() > 256 + || scope.cluster.contains('\0') + || !valid_uuid(scope.node_incarnation) + || !valid_uuid(scope.start_request) + || !valid_uuid(scope.env_generation) + || scope.env_keys.len() > MAX_ENV_KEYS + || scope.env_keys.windows(2).any(|keys| keys[0] >= keys[1]) + || scope.env_keys.iter().any(|key| validate_env_key(key).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidScope); + } + Ok(()) +} + +fn fence( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + state + .iter_by_col_eq( + ST_CONTAINER_FENCE_ID, + ColId(0), + &AlgebraicValue::U256(scope.database_identity.to_u256().into()), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerFenceRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::Fenced) +} + +fn admitted_spec( + db: &RelationalDB, + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + validate_scope(db, scope)?; + let current = fence(state, scope)?; + if current.generation != scope.generation || !current.allowed { + return Err(EnvironmentSnapshotError::Fenced); + } + let (revision, deployment) = deployment::current_deployment(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + let publication = deployment::current_publication(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if revision != scope.deployment_revision + || publication.operation_id != scope.publication_operation + || publication.publication_epoch != scope.publication_epoch + { + return Err(EnvironmentSnapshotError::RevisionConflict); + } + let spec = deployment + .current() + .container + .as_ref() + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if spec.env_keys != scope.env_keys { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + Ok(spec.clone()) +} + +fn lookup( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result, EnvironmentSnapshotError> { + let Some(row) = state + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerEnvironmentRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + else { + return Ok(None); + }; + if row.payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + let Record::V1(record) = bsatn::from_slice(&row.payload).map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if record.receipt.scope != *scope { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + if !valid_uuid(record.receipt.capture_receipt) + || !record + .selected_values + .iter() + .map(|entry| &entry.key) + .eq(scope.env_keys.iter()) + { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + Ok(Some(record)) +} + +fn validate_values(spec: &ContainerSpec, values: &BTreeMap) -> Result<(), EnvironmentSnapshotError> { + // The at-most-256 individually bounded values also bound this temporary allocation. + if values + .values() + .any(|value| value.contains('\0') || spacetimedb_lib::environment::validate_value(value).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidEnvironment); + } + let env = values + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>(); + validate_exec_size(&spec.argv, &env).map_err(|_| EnvironmentSnapshotError::InvalidEnvironment) +} + +/// Capture exactly once under an open confirmed control intent. Returning this +/// receipt is not a durability acknowledgment; the host wrapper supplies that. +pub fn capture( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, +) -> Result { + let spec = admitted_spec(db, tx, scope)?; + if let Some(record) = lookup(tx, scope)? { + validate_values( + &spec, + &record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(), + )?; + return Ok(record.receipt); + } + if tx + .table_row_count(ST_CONTAINER_ENVIRONMENT_ID) + .ok_or(EnvironmentSnapshotError::Storage)? + >= MAX_RETAINED_SNAPSHOTS + { + return Err(EnvironmentSnapshotError::Capacity); + } + let mut values = BTreeMap::new(); + let mut missing = Vec::new(); + for key in &scope.env_keys { + match environment::get(tx, key).map_err(|_| EnvironmentSnapshotError::Storage)? { + Some(value) => { + values.insert(key.clone(), value); + } + None => missing.push(key.clone()), + } + } + if !missing.is_empty() { + return Err(EnvironmentSnapshotError::MissingKeys(missing)); + } + validate_values(&spec, &values)?; + let receipt = EnvironmentSnapshotReceipt { + scope: scope.clone(), + capture_receipt: Uuid::from_u128(uuid::Uuid::new_v4().as_u128()), + }; + let payload = bsatn::to_vec(&Record::V1(RecordV1 { + receipt: receipt.clone(), + selected_values: values + .into_iter() + .map(|(key, value)| CapturedValue { key, value }) + .collect(), + })) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::Capacity); + } + tx.insert_via_serialize_bsatn( + ST_CONTAINER_ENVIRONMENT_ID, + &StContainerEnvironmentRow { + generation: scope.generation, + payload: payload.into(), + }, + ) + .map_err(|_| EnvironmentSnapshotError::Storage)?; + Ok(receipt) +} + +/// Read only an existing receipt after fresh exact control/lease confirmation. +pub fn read( + db: &RelationalDB, + state: &impl StateView, + receipt: &EnvironmentSnapshotReceipt, +) -> Result { + let spec = admitted_spec(db, state, &receipt.scope)?; + let record = lookup(state, &receipt.scope)?.ok_or(EnvironmentSnapshotError::NotCaptured)?; + if record.receipt != *receipt { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + let selected_values = record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(); + validate_values(&spec, &selected_values)?; + Ok(SecretEnvironment { + receipt: record.receipt, + selected_values, + }) +} + +/// Delete only after a newer own-source fence irreversibly rejects every old +/// capture. The adapter also confirms positive historical control closure. +/// Missing rows remain idempotently closed; no per-UUID TTL is an authority. +pub fn close( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + validate_scope(db, scope)?; + let current = fence(tx, scope)?; + if closed_through_generation <= scope.generation || current.generation < closed_through_generation { + return Err(EnvironmentSnapshotError::Fenced); + } + if lookup(tx, scope)?.is_some() { + let pointer = tx + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .ok_or(EnvironmentSnapshotError::Storage)? + .pointer(); + db.delete(tx, ST_CONTAINER_ENVIRONMENT_ID, [pointer]); + } + Ok(EnvironmentClosedReceipt { + scope: scope.clone(), + closed_through_generation, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/container_environment/tests.rs b/crates/core/src/db/container_environment/tests.rs new file mode 100644 index 00000000000..ec300aecf80 --- /dev/null +++ b/crates/core/src/db/container_environment/tests.rs @@ -0,0 +1,579 @@ +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::tests_utils::TestDB; +use crate::host::container_environment as host; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent}; +use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use spacetimedb_lib::{hash_bytes, Identity, Timestamp}; +use std::sync::{Arc, Barrier}; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn environment_schema(keys: &[String]) -> EnvironmentSchema { + EnvironmentSchema::new( + keys.iter() + .map(|name| EnvironmentDeclaration { + name: name.clone(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }) + .collect(), + ) + .unwrap() +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let schema = environment_schema(&spec.env_keys); + let generated = spacetimedb_lib::deployment::system_empty::generate(&schema).unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + expected_last_operation: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(generated.descriptor), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + environment::replace( + db, + tx, + &schema, + &spec.env_keys.iter().map(|key| (key.clone(), "before".into())).collect(), + )?; + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + publication_operation: request.operation_id, + publication_epoch: request.publication_epoch, + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +fn tx( + db: &RelationalDB, + action: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, action) +} + +#[test] +fn container_environment_capture_retry_read_and_durable_reopen() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope.clone())) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "changed".into())]), + )?; + Ok(()) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let retried = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope)) + .unwrap(); + assert_eq!(retried.receipt, captured.receipt); + assert!(retried.durable_through >= captured.durable_through); + let values = db + .runtime() + .unwrap() + .block_on(host::read(db.db.clone(), retried.receipt)) + .unwrap() + .receipt; + assert_eq!( + values.selected_values, + BTreeMap::from([("A".into(), "before".into()), ("B".into(), "before".into())]) + ); + assert!(!format!("{values:?}").contains("before")); +} + +#[test] +fn container_environment_capture_is_atomic_against_concurrent_environment_mutation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let barrier = Arc::new(Barrier::new(2)); + let writer = { + let db = db.db.clone(); + let barrier = barrier.clone(); + let schema = environment_schema(&scope.env_keys); + std::thread::spawn(move || { + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::replace( + &db, + tx, + &schema, + &BTreeMap::from([("A".into(), "after".into()), ("B".into(), "after".into())]), + )?; + Ok(()) + }) + .unwrap(); + }) + }; + barrier.wait(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + writer.join().unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], values.selected_values["B"]); +} + +#[test] +fn container_environment_closure_fences_delayed_capture_and_new_instance_observes_new_values() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let old = setup(&db, vec!["A".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), old.clone())) + .unwrap() + .receipt; + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 1)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 2)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + environment::replace( + &db, + tx, + &environment_schema(&old.env_keys), + &BTreeMap::from([("A".into(), "new-boot".into())]), + )?; + Ok(()) + }) + .unwrap(); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let db = db.reopen().unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &old)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &captured)), + Err(EnvironmentSnapshotError::Fenced) + )); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let new = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..old + }; + let receipt = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), new)) + .unwrap() + .receipt; + assert_ne!(receipt.capture_receipt, captured.capture_receipt); + assert_eq!( + db.runtime() + .unwrap() + .block_on(host::read(db.db.clone(), receipt)) + .unwrap() + .receipt + .selected_values["A"], + "new-boot" + ); +} + +#[test] +fn container_environment_full_scope_conflicts_and_missing_ready_record_fail_closed() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let mut variants = Vec::new(); + let mut changed = scope.clone(); + changed.cluster = "other".into(); + variants.push(changed); + let mut changed = scope.clone(); + changed.database_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_incarnation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.start_request = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_generation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_keys.clear(); + variants.push(changed); + for changed in variants { + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::ScopeConflict) + ); + } + let mut changed = scope.clone(); + changed.deployment_revision = hash_bytes(b"other"); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + let mut fork = scope; + fork.database_identity = Identity::from_u256(99u64.into()); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &fork)), + Err(EnvironmentSnapshotError::InvalidScope) + ); + // Simulate a restored/lost Ready record. Resolve may not reinterpret its UUID. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + tx.clear_table(ST_CONTAINER_ENVIRONMENT_ID)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::NotCaptured) + )); +} + +#[test] +fn container_environment_missing_empty_invalid_and_capacity_are_atomic_and_redacted() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "before".into())]), + )?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::MissingKeys(vec!["B".into()])) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "before".into()), ("B".into(), "secret\0value".into())]), + )?; + Ok(()) + }) + .unwrap(); + let failure = tx(&db, |tx| capture(&db, tx, &scope)).unwrap_err(); + assert_eq!(failure, EnvironmentSnapshotError::InvalidEnvironment); + assert!(!format!("{failure:?}: {failure}").contains("secret")); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "before".into()), ("B".into(), "".into())]), + )?; + Ok(()) + }) + .unwrap(); + for generation in 1..=MAX_RETAINED_SNAPSHOTS { + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + let request = EnvironmentSnapshotScope { + generation, + env_generation: uuid(), + ..scope.clone() + }; + let receipt = tx(&db, |tx| capture(&db, tx, &request)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["B"], + "" + ); + } + let generation = MAX_RETAINED_SNAPSHOTS + 1; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture( + &db, + tx, + &EnvironmentSnapshotScope { generation, ..scope } + )), + Err(EnvironmentSnapshotError::Capacity) + ); +} + +#[test] +fn container_environment_history_is_hidden_from_privileged_sql_and_subscriptions() { + use crate::db::sql::ast::SchemaViewer; + use spacetimedb_expr::check::SchemaView; + use spacetimedb_lib::identity::AuthCtx; + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let auth = AuthCtx::for_current(db.owner_identity()); + db.with_read_only(Workload::ForTests, |state| { + let schema = SchemaViewer::new(state, &auth); + assert!(schema.schema_for_table(ST_CONTAINER_ENVIRONMENT_ID).is_none()); + assert!(schema.table_id("st_container_environment").is_none()); + for sql in [ + "SELECT * FROM st_container_environment", + "SELECT h.* FROM st_container_environment h JOIN st_env e ON h.generation = 1", + "DELETE FROM st_container_environment", + "UPDATE st_container_environment SET generation = 2", + ] { + assert!( + spacetimedb_query::compile_sql_stmt(sql, &schema, &auth).is_err(), + "{sql}" + ); + } + assert!(spacetimedb_query::compile_sql_stmt("SELECT * FROM st_env", &schema, &auth).is_ok()); + assert!(crate::subscription::query::compile_read_only_query( + &auth, + state, + "SELECT * FROM st_container_environment" + ) + .is_err()); + assert!(crate::subscription::subscription::get_all( + |db, tx| db.get_all_tables(tx).map(Vec::into_iter), + &db, + state, + &auth + ) + .unwrap() + .is_empty()); + }); +} + +#[test] +fn container_environment_host_api_requires_durability() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec![]); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + assert!(matches!( + rt.block_on(host::capture(db.db.clone(), scope)), + Err(EnvironmentSnapshotError::DurabilityUnavailable) + )); +} + +#[test] +fn container_environment_concurrent_closure_cannot_reopen_collected_generation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let barrier = Arc::new(Barrier::new(2)); + let capture_thread = { + let db = db.db.clone(); + let scope = scope.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + tx(&db, |tx| capture(&db, tx, &scope)) + }) + }; + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, false))?; + close(&db, tx, &scope, 2)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + capture_thread.join().unwrap(), + Ok(_) | Err(EnvironmentSnapshotError::Fenced) + )); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_read_only(Workload::ForTests, |state| { + assert_eq!(state.table_row_count(ST_CONTAINER_ENVIRONMENT_ID), Some(0)) + }); +} + +#[test] +fn snapshots_bind_committed_operation_and_ignore_newer_aborted_attempt_epoch() { + use crate::db::deployment::{abort_deployment_commit, current_publication}; + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let original = setup(&db, vec!["A".into()]); + let attempt = db + .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + let current = current_publication(tx)?.unwrap(); + let attempt = DeploymentCommit { + operation_id: uuid(), + publication_epoch: current.publication_epoch + 1, + publisher: db.owner_identity(), + expected_revision: Some(current.revision), + expected_last_operation: Some(current.operation_id), + prepared_manifest_hash: hash_bytes(b"aborted"), + deployment: deployment::current_deployment(tx)?.unwrap().1, + }; + install_publication_fence(tx, attempt.publication_epoch, attempt.operation_id)?; + abort_deployment_commit(tx, &attempt)?; + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + Ok(attempt) + }) + .unwrap(); + let scope = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..original + }; + // Real commitlog replay retains the committed operation and an unrelated + // later closed attempt. A fresh generation still captures the old values. + let db = db.reopen().unwrap(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], "before"); + let mut wrong = scope.clone(); + wrong.publication_epoch = attempt.publication_epoch; + assert_eq!( + tx(&db, |tx| capture(&db, tx, &wrong)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + wrong.publication_operation = attempt.operation_id; + assert_eq!( + tx(&db, |tx| capture(&db, tx, &wrong)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + + let accepted = db + .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + let mut next = attempt.clone(); + next.operation_id = uuid(); + next.publication_epoch += 1; + install_publication_fence(tx, next.publication_epoch, next.operation_id)?; + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "new-secret".into())]), + )?; + Ok(record_deployment_commit( + tx, + &next, + Timestamp::now(), + &Default::default(), + )?) + }) + .unwrap(); + assert_eq!(accepted.revision, scope.deployment_revision); + // Keep the generation fence unchanged here to prove the publication cursor + // independently blocks both old capture and old immutable snapshot reads. + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::RevisionConflict) + )); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 3, true))?; + close(&db, tx, &scope, 3)?; + Ok(()) + }) + .unwrap(); + let next_scope = EnvironmentSnapshotScope { + generation: 3, + env_generation: uuid(), + publication_operation: accepted.operation_id, + publication_epoch: accepted.publication_epoch, + ..scope + }; + let receipt = tx(&db, |tx| capture(&db, tx, &next_scope)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["A"], + "new-secret" + ); +} diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs new file mode 100644 index 00000000000..f31753b62c5 --- /dev/null +++ b/crates/core/src/db/deployment.rs @@ -0,0 +1,623 @@ +//! Transactional deployment and hosted-client fences. +//! +//! Only authenticated host operations may call the mutation functions here. +//! Callers retain the same serializable transaction through module migration, +//! deployment recording, and commit. These functions do not contact control, +//! perform process IO, or turn an unverified client Identity into host authority. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + ConnectionIdViaU128, StConnectionAuthRow, StContainerFenceRow, StDeploymentOperationRow, StDeploymentRow, + StPublishFenceRow, ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, + ST_PUBLISH_FENCE_ID, +}; +use spacetimedb_lib::container::ContainerSpecLimits; +use spacetimedb_lib::deployment::{operation_expiry_ms, DeploymentSpec, DeploymentValidationError}; +use spacetimedb_lib::{bsatn, hash_bytes, ConnectionId, Hash, Identity, SpacetimeType, Timestamp, Uuid}; +use spacetimedb_primitives::{ColId, TableId}; +use spacetimedb_sats::AlgebraicValue; + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentError { + #[error("this database requires the deployment publication protocol")] + CoordinatorRequired, + #[error("the supplied module does not match the prepared deployment")] + ProgramMismatch, + #[error("the prepared module must advertise hosted_auth_v1 to attach a container")] + UnsupportedHostedModule, + #[error("the publication coordinator no longer owns the database fence")] + PublicationFenced, + #[error("the expected committed publication does not match the database")] + RevisionConflict, + #[error("operation ID is already bound to a different publication")] + OperationConflict, + #[error("container generation or grant is not authorized by this database")] + ContainerFenced, + #[error("a conflicting or older container fence cannot replace current authority")] + FenceConflict, + #[error("the receiving host fence revision is exhausted")] + FenceRevisionExhausted, + #[error("deployment metadata is inconsistent")] + CorruptMetadata, + #[error(transparent)] + Validation(#[from] DeploymentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Prepared by the authorized coordinator after recording durable intent. +/// The publisher is its verified original caller, never a guest-selected claim. +#[derive(Clone, Debug)] +pub struct DeploymentCommit { + pub operation_id: Uuid, + pub publication_epoch: u64, + pub publisher: Identity, + pub expected_revision: Option, + pub expected_last_operation: Option, + pub prepared_manifest_hash: Hash, + pub deployment: DeploymentSpec, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishResult { + #[serde(with = "spacetimedb_lib::deployment::uuid_json")] + pub operation_id: Uuid, + pub publication_epoch: u64, + pub previous_revision: Option, + pub revision: Hash, +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum CommitReceipt { + V1(CommitReceiptV1), +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CommitReceiptV1 { + request_hash: Hash, + publisher: Identity, + result: PublishResult, +} + +#[derive(Clone, Debug)] +pub enum CommitAdmission { + /// Return this result without repeating module init/update or any effects. + AlreadyCommitted(PublishResult), + Ready, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AbortResult { + /// The commit won the race. Recovery must converge on this deployment. + AlreadyCommitted(PublishResult), + /// This epoch can no longer admit a commit, and the prior revision remains. + Aborted { previous_revision: Option }, +} + +/// Check the actual program selected by the host, not a caller-provided +/// capability bit. The program bytes are hashed here because `Program` also +/// has a public constructor which accepts a previously computed hash. +pub fn validate_deployment_program( + request: &DeploymentCommit, + program: &spacetimedb_datastore::traits::Program, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + use spacetimedb_datastore::system_tables::ModuleKind; + use spacetimedb_lib::deployment::{ModuleComponent, UserModuleKind}; + if hash_bytes(&program.bytes) != program.hash { + return Err(DeploymentError::ProgramMismatch); + } + match &request.deployment.current().module { + ModuleComponent::User(expected) + if expected.program_hash == program.hash + && matches!( + (expected.kind, program.kind), + (UserModuleKind::Wasm, ModuleKind::WASM) | (UserModuleKind::Js, ModuleKind::JS) + ) => {} + ModuleComponent::SystemEmpty(module) if crate::host::empty_module::matches_program(module, program) => {} + _ => return Err(DeploymentError::ProgramMismatch), + } + if request.deployment.current().container.is_some() && !module.supports_hosted_auth_v1() { + return Err(DeploymentError::UnsupportedHostedModule); + } + Ok(()) +} + +/// Legacy raw module publication must be routed through the coordinator once +/// a database has deployment metadata or a publication fence, including after +/// an attempt aborts or its container is removed. In particular a legacy +/// request cannot race the first prepared container publication. +/// Call while holding the transaction that changes the program/schema. +pub fn require_unmanaged_publication(tx: &MutTx) -> Result<(), DeploymentError> { + if current_deployment(tx)?.is_some() || singleton(tx, ST_PUBLISH_FENCE_ID)?.is_some() { + return Err(DeploymentError::CoordinatorRequired); + } + Ok(()) +} + +/// Foreign callers depend on the receiving module's bindings just as self +/// callers do. A module replacement cannot silently erase that capability +/// while an admitted generation can still address the database. +pub fn validate_active_hosted_grants( + tx: &MutTx, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + if !module.supports_hosted_auth_v1() { + for row in tx.iter(ST_CONTAINER_FENCE_ID)? { + if StContainerFenceRow::try_from(row)?.allowed { + return Err(DeploymentError::UnsupportedHostedModule); + } + } + } + Ok(()) +} + +fn singleton( + state: &S, + table: TableId, +) -> Result>, DeploymentError> { + Ok(state.iter_by_col_eq(table, ColId(0), &AlgebraicValue::U8(0))?.next()) +} + +pub fn current_deployment(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + if spec.revision()? != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some((row.revision, spec))) +} + +/// The current committed cursor comes from its exact retained receipt, never +/// from the attempt fence (which may belong to a later aborted publication). +/// Keep this receipt while its operation is current, even after retry expiry. +pub fn current_publication(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + let operation = Uuid::from_u128(row.last_operation_id); + let receipt = retained_publication(state, operation)?.ok_or(DeploymentError::CorruptMetadata)?; + if spec.revision()? != row.revision || receipt.result.revision != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some(receipt.result)) +} + +fn require_expected_publication(state: &S, request: &DeploymentCommit) -> Result<(), DeploymentError> { + let current = current_publication(state)?; + if current.as_ref().map(|result| result.revision) != request.expected_revision + || current.as_ref().map(|result| result.operation_id) != request.expected_last_operation + { + return Err(DeploymentError::RevisionConflict); + } + Ok(()) +} + +fn retained_publication(state: &S, operation: Uuid) -> Result, DeploymentError> { + let operation_key = AlgebraicValue::U128(operation.as_u128().into()); + let Some(row) = state + .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? + .next() + else { + return Ok(None); + }; + let row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(receipt) = + bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if operation.get_version() != Some(spacetimedb_sats::uuid::Version::V7) + || row.operation_id != operation.as_u128() + || receipt.result.operation_id != operation + || receipt.result.publication_epoch == 0 + || row.committed_revision != receipt.result.revision + || row.previous_revision != receipt.result.previous_revision + { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some(receipt)) +} + +/// Monotonic compare-and-set, serialized against user-database commits. +/// Advancing this epoch does not itself quiesce a container or authorize launch. +pub fn install_publication_fence( + tx: &mut MutTx, + publication_epoch: u64, + operation_id: Uuid, +) -> Result<(), DeploymentError> { + if publication_epoch == 0 || operation_id == Uuid::NIL { + return Err(DeploymentError::PublicationFenced); + } + let next = StPublishFenceRow { + key: 0, + publication_epoch, + operation_id: operation_id.as_u128(), + }; + if let Some(current) = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()? + { + if current == next { + return Ok(()); + } + if current.publication_epoch >= publication_epoch { + return Err(DeploymentError::PublicationFenced); + } + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn(ST_PUBLISH_FENCE_ID, &next)?; + Ok(()) +} + +/// Close an admitted publication before reporting an abort to control. Run in +/// one serializable transaction and await its durability before releasing the +/// control operation or resuming the previous container under a fresh generation. +/// A delayed commit and this transaction serialize on the same database fence. +/// +/// The nil operation ID is a closed-epoch marker, never a publish operation. +/// Keeping the epoch makes closure irreversible at that epoch while allowing +/// the next control-allocated epoch to install its own operation normally. +pub fn abort_deployment_commit(tx: &mut MutTx, request: &DeploymentCommit) -> Result { + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(AbortResult::AlreadyCommitted(result)); + } + // Recovery must be able to close an expired operation too. Its expiry + // prevents new commits, but cannot substitute for a durable abort fence. + // Control owns the immutable epoch-to-operation mapping; authenticate and + // resolve that recorded operation before calling this function. + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + let fence = fence + .filter(|row| { + row.publication_epoch == request.publication_epoch + && (row.operation_id == request.operation_id.as_u128() || row.operation_id == 0) + }) + .ok_or(DeploymentError::PublicationFenced)?; + require_expected_publication(tx, request)?; + if fence.operation_id == 0 { + return Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }); + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn( + ST_PUBLISH_FENCE_ID, + &StPublishFenceRow { + key: 0, + publication_epoch: request.publication_epoch, + operation_id: 0, + }, + )?; + Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }) +} + +fn normalized_request( + request: &DeploymentCommit, + limits: &ContainerSpecLimits, +) -> Result<(DeploymentSpec, Hash, Hash), DeploymentError> { + let spec = request.deployment.clone().normalize(limits)?; + if spec != request.deployment { + return Err(DeploymentValidationError::InvalidEncoding.into()); + } + let (revision, request_hash) = request_identity(request)?; + Ok((spec, revision, request_hash)) +} + +fn request_identity(request: &DeploymentCommit) -> Result<(Hash, Hash), DeploymentError> { + if request.operation_id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId.into()); + } + if request.publication_epoch == 0 { + return Err(DeploymentError::PublicationFenced); + } + // These bytes were normalized at admission. Do not re-apply today's + // resource eligibility when inspecting yesterday's committed outcome. + let revision = request.deployment.revision()?; + let mut bytes = b"spacetimedb/deployment-operation\0".to_vec(); + bytes.extend_from_slice( + &bsatn::to_vec(&( + request.operation_id, + request.publication_epoch, + request.publisher, + request.expected_revision, + request.expected_last_operation, + request.prepared_manifest_hash, + revision, + )) + .map_err(|_| DeploymentError::CorruptMetadata)?, + ); + Ok((revision, hash_bytes(bytes))) +} + +/// Call before module execution, while holding the transaction later used for +/// migration. A cache or pre-enqueue check cannot replace this admission check. +pub fn check_deployment_commit( + tx: &MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + let now_ms = u64::try_from(now.to_micros_since_unix_epoch()).map_err(|_| DeploymentError::CorruptMetadata)? / 1000; + operation_expiry_ms(request.operation_id, now_ms)?; + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(CommitAdmission::AlreadyCommitted(result)); + } + normalized_request(request, limits)?; + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|f| { + f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() + }) { + return Err(DeploymentError::PublicationFenced); + } + require_expected_publication(tx, request)?; + Ok(CommitAdmission::Ready) +} + +/// Inspect the exact retained commit receipt during host recovery. Unlike +/// admitting a client retry, inspecting an existing outcome does not expire. +/// This never authorizes module execution. An absent receipt is not proof of +/// abort: close the epoch atomically before reporting an abort to control. +/// Retain active operations' receipts until their control recovery completes. +pub fn committed_deployment_operation( + state: &S, + request: &DeploymentCommit, +) -> Result, DeploymentError> { + let (revision, request_hash) = request_identity(request)?; + if let Some(receipt) = retained_publication(state, request.operation_id)? { + if receipt.request_hash != request_hash || receipt.publisher != request.publisher { + return Err(DeploymentError::OperationConflict); + } + if receipt.result.revision != revision || receipt.result.publication_epoch != request.publication_epoch { + return Err(DeploymentError::CorruptMetadata); + } + return Ok(Some(receipt.result)); + } + Ok(None) +} + +/// Recognize the durable closed marker when recovering a control operation +/// whose abort report was lost. The authenticated caller must resolve control's +/// immutable epoch-to-operation binding before using this host-only API. +pub fn deployment_publication_aborted( + state: &S, + request: &DeploymentCommit, +) -> Result { + request_identity(request)?; + let fence = singleton(state, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|row| row.publication_epoch == request.publication_epoch && row.operation_id == 0) { + return Ok(false); + } + require_expected_publication(state, request)?; + Ok(true) +} + +/// Record after successful module initialization/migration in that same +/// transaction. An error must roll back the entire transaction, including the +/// module changes. The caller waits for durability before reporting acceptance. +pub fn record_deployment_commit( + tx: &mut MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + if let CommitAdmission::AlreadyCommitted(result) = check_deployment_commit(tx, request, now, limits)? { + return Ok(result); + } + let (spec, revision, request_hash) = normalized_request(request, limits)?; + let result = PublishResult { + operation_id: request.operation_id, + publication_epoch: request.publication_epoch, + previous_revision: request.expected_revision, + revision, + }; + let receipt = CommitReceipt::V1(CommitReceiptV1 { + request_hash, + publisher: request.publisher, + result: result.clone(), + }); + let receipt = bsatn::to_vec(&receipt) + .map_err(|_| DeploymentError::CorruptMetadata)? + .into_boxed_slice(); + let expires_ms = operation_expiry_ms(request.operation_id, (now.to_micros_since_unix_epoch() / 1000) as u64)?; + let expires_us = i64::try_from(expires_ms * 1000).map_err(|_| DeploymentError::CorruptMetadata)?; + let row = StDeploymentRow { + key: 0, + revision, + last_operation_id: request.operation_id.as_u128(), + payload: spec.encode()?, + }; + tx.clear_table(ST_DEPLOYMENT_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_ID, &row)?; + tx.insert_via_serialize_bsatn( + ST_DEPLOYMENT_OPERATION_ID, + &StDeploymentOperationRow { + operation_id: request.operation_id.as_u128(), + previous_revision: request.expected_revision, + committed_revision: revision, + commit_result: receipt, + expires_at: Timestamp::from_micros_since_unix_epoch(expires_us).into(), + }, + )?; + Ok(result) +} + +/// Install the frozen generation/grant tuple. Changes at the same generation +/// cannot reopen a revoked grant, even if a stale coordinator changes only the +/// target-set hash. Control must allocate a new generation for every barrier. +pub fn install_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + next: &StContainerFenceRow, +) -> Result<(), DeploymentError> { + if next.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = next.source_identity.into(); + let previous = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()?; + if let Some((pointer, previous)) = previous { + if previous == *next { + return Ok(()); + } + if next.generation <= previous.generation || next.target_grant_revision < previous.target_grant_revision { + return Err(DeploymentError::FenceConflict); + } + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + } else { + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + } + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, next)?; + Ok(()) +} + +/// An exact denial of a previously observed allowed fence. Revision counters +/// belong to this live database open, not to the durable generation namespace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FenceDenial { + pub row: StContainerFenceRow, + pub revision_before: u64, + pub revision_after: u64, +} + +/// Deny an orphan without lowering or inventing a control generation. +/// +/// The trusted coordinator must establish the source's absence from current +/// control inventory, then recheck that inventory revision while holding this +/// same serializable transaction. This function does not confer authority to +/// any external caller. Only the exact allowed tuple, or its already-denied +/// form, is accepted. Ordinary installation cannot reopen this generation. +pub fn deny_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + expected: &StContainerFenceRow, +) -> Result { + if !expected.allowed || expected.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = expected.source_identity.into(); + let (pointer, current) = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()? + .ok_or(DeploymentError::FenceConflict)?; + let denied = StContainerFenceRow { + allowed: false, + ..expected.clone() + }; + let (revision_before, revision_after) = if current == denied { + let revision = db.hosted_admission().fence_revision(); + (revision, revision) + } else { + if current != *expected { + return Err(DeploymentError::FenceConflict); + } + // The gate is closed before the row changes. A rollback still + // conservatively invalidates any concurrently progressing page scan. + let revisions = db + .hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, &denied)?; + revisions + }; + Ok(FenceDenial { + row: denied, + revision_before, + revision_after, + }) +} + +/// Called with verified hosted credentials inside every admitted transaction, +/// including each later transaction of a procedure. Signature, audience, expiry, +/// capability, and interface checks are additional receiving-host requirements. +pub fn check_container_fence( + state: &S, + source: Identity, + generation: u64, + target_grant_revision: u64, +) -> Result<(), DeploymentError> { + let key = AlgebraicValue::U256(source.to_u256().into()); + let fence = state + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(StContainerFenceRow::try_from) + .transpose()?; + if !fence + .is_some_and(|f| f.allowed && f.generation == generation && f.target_grant_revision == target_grant_revision) + { + return Err(DeploymentError::ContainerFenced); + } + Ok(()) +} + +/// Capture validated hosted connection authority in the transaction inserting st_client. +/// This is host-only metadata; ordinary connections leave no row and retain flags zero. +pub(crate) fn record_connection_auth( + tx: &mut MutTx, + connection_id: ConnectionId, + sender: Identity, + call_auth_flags: u32, +) -> Result<(), DeploymentError> { + tx.insert_via_serialize_bsatn( + ST_CONNECTION_AUTH_ID, + &StConnectionAuthRow { + connection_id: connection_id.into(), + sender_identity: sender.into(), + call_auth_flags, + }, + )?; + Ok(()) +} + +/// Recover captured authority for a host-dispatched disconnect event. It is not +/// a new container admission and does not require a still-valid credential/lease. +pub(crate) fn connection_auth_flags( + state: &S, + connection_id: ConnectionId, + sender: Identity, +) -> Result { + let key: AlgebraicValue = ConnectionIdViaU128::from(connection_id).into(); + let row = state + .iter_by_col_eq(ST_CONNECTION_AUTH_ID, ColId(0), &key)? + .next() + .map(StConnectionAuthRow::try_from) + .transpose()?; + let Some(row) = row else { return Ok(0) }; + if row.sender_identity.0 != sender { + return Err(DeploymentError::CorruptMetadata); + } + Ok(row.call_auth_flags) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs new file mode 100644 index 00000000000..e031319d575 --- /dev/null +++ b/crates/core/src/db/deployment/tests.rs @@ -0,0 +1,791 @@ +use super::*; +use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::{ + StEnvRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_ENV_ID, +}; +use spacetimedb_durability::Durability; +use spacetimedb_lib::deployment::{ + DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind, PUBLISH_RETRY_WINDOW_MS, +}; + +fn request(sequence: u64, previous: Option<&PublishResult>) -> DeploymentCommit { + DeploymentCommit { + operation_id: Uuid::from_u128(0x01991ec4000070008000000000000000 | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::from_u256(55u64.into()), + expected_revision: previous.map(|result| result.revision), + expected_last_operation: previous.map(|result| result.operation_id), + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(sequence.to_le_bytes()), + }), + container: None, + }), + } +} + +fn now() -> Timestamp { + Timestamp::from_micros_since_unix_epoch(((request(1, None).operation_id.as_u128() >> 80) as i64) * 1000) +} + +fn transact( + db: &RelationalDB, + f: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, f) +} + +#[test] +fn deployment_retry_returns_original_result_after_later_publish_without_mutation() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(&accepted)); + let later = transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!(check_deployment_commit(tx, &first, now(), &limits)?, CommitAdmission::AlreadyCommitted(ref r) if r == &accepted)); + assert_eq!(record_deployment_commit(tx, &first, now(), &limits)?, accepted); + assert_eq!(current_deployment(tx)?.unwrap().0, later.revision); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let mut second = request(2, None); + second.expected_revision = Some(hash_bytes(b"not current")); + second.expected_last_operation = Some(first.operation_id); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + second.expected_revision = None; + second.expected_last_operation = None; + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn deployment_and_module_effects_roll_back_together() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + // A second persistent table stands in for migration effects in the same + // transaction. Integration must additionally execute Wasm/JS migrations. + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "MIGRATED".into(), + value: "yes".into(), + }, + )?; + record_deployment_commit(tx, &request, now(), &limits)?; + Err(DeploymentError::Database(DBError::Other(anyhow::anyhow!( + "injected failure before commit" + )))) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(current_deployment(tx)?.is_none()); + assert_eq!(tx.table_row_count(ST_ENV_ID), Some(0)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(0)); + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_closes_delayed_commits_and_cannot_reopen_its_epoch() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id) + }) + .unwrap(); + let aborted = AbortResult::Aborted { + previous_revision: None, + }; + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &first)).unwrap(), + aborted + ); + transact(&db, |tx| { + assert_eq!(abort_deployment_commit(tx, &first)?, aborted); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch + 1, Uuid::NIL), + Err(DeploymentError::PublicationFenced) + )); + assert!(current_deployment(tx)?.is_none()); + let second = request(2, None); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + let result = record_deployment_commit(tx, &second, now(), &limits)?; + assert!(matches!( + abort_deployment_commit(tx, &first), + Err(DeploymentError::PublicationFenced) + )); + assert_eq!(current_deployment(tx)?.unwrap().0, result.revision); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_commit_winning_abort_race_is_never_reported_as_aborted() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(&committed)); + transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits)?; + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed) + ); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::AlreadyCommitted(_) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_rollback_does_not_report_a_closed_fence() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + abort_deployment_commit(tx, &request)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn recovery_of_expired_publication_preserves_commits_and_closes_uncommitted_epochs() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS * 1000) as i64, + ); + let second = request(2, Some(&committed)); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(current_publication(tx)?, Some(committed.clone())); + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + let mut wrong_publisher = first.clone(); + wrong_publisher.publisher = Identity::ZERO; + assert!(matches!( + committed_deployment_operation(tx, &wrong_publisher), + Err(DeploymentError::OperationConflict) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &second, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!( + abort_deployment_commit(tx, &second)?, + AbortResult::Aborted { + previous_revision: Some(committed.revision) + } + ); + // Clock rollback cannot make a delayed old commit admissible again. + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_and_commit_receipts_survive_commitlog_replay() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + abort_deployment_commit(tx, &first) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let second = request(2, None); + let committed = transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::Aborted { + previous_revision: None + } + ); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + let db = db.reopen().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + committed_deployment_operation(tx, &second).unwrap(), + Some(committed.clone()) + ); + assert_eq!(current_deployment(tx).unwrap().unwrap().0, committed.revision); + }); + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &second)).unwrap(), + AbortResult::AlreadyCommitted(committed) + ); +} + +#[test] +fn publication_recovery_does_not_reapply_tightened_resource_admission() { + use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, + }; + let db = TestDB::in_memory().unwrap(); + let mut first = request(1, None); + let DeploymentSpec::V1(spec) = &mut first.deployment; + spec.container = Some(ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 10_000, + }); + let original_limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &original_limits) + }) + .unwrap(); + let mut tightened = original_limits; + tightened.resources.cpu_millicores = 500; + let mut second = request(2, Some(&committed)); + second.deployment = first.deployment.clone(); + transact(&db, |tx| { + assert!(first.deployment.clone().normalize(&tightened).is_err()); + assert!(matches!(check_deployment_commit(tx, &first, now(), &tightened)?, CommitAdmission::AlreadyCommitted(ref result) if result == &committed)); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(abort_deployment_commit(tx, &first)?, AbortResult::AlreadyCommitted(committed.clone())); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(check_deployment_commit(tx, &second, now(), &tightened).is_err()); + assert_eq!(abort_deployment_commit(tx, &second)?, AbortResult::Aborted { previous_revision: Some(committed.revision) }); + assert!(deployment_publication_aborted(tx, &second)?); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_operation_cannot_be_reused_by_another_publisher_or_changed_request() { + let db = TestDB::in_memory().unwrap(); + let original = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, original.publication_epoch, original.operation_id)?; + record_deployment_commit(tx, &original, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = original.clone(); + changed.publisher = Identity::from_u256(99u64.into()); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + changed = original.clone(); + changed.deployment = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(spacetimedb_lib::deployment::system_empty::empty().descriptor), + container: None, + }); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + // A retained row does not extend the advertised retry window. + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS as i64) * 1000, + ); + assert!(matches!( + check_deployment_commit(tx, &original, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_revokes_copied_credentials_and_does_not_reopen_at_same_generation() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(777u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"targets1"), + allowed: true, + }; + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + install_container_fence(&db, tx, &first)?; + install_container_fence(&db, tx, &first)?; + check_container_fence(tx, source, 1, 3)?; + assert!(matches!( + check_container_fence(tx, source, 1, 4), + Err(DeploymentError::ContainerFenced) + )); + Ok(()) + }) + .unwrap(); + let revoked = StContainerFenceRow { + generation: 2, + target_grant_revision: 4, + target_set_hash: hash_bytes(b"targets2"), + allowed: false, + ..first.clone() + }; + transact(&db, |tx| { + install_container_fence(&db, tx, &revoked)?; + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + check_container_fence(tx, source, 2, 4), + Err(DeploymentError::ContainerFenced) + )); + let reopen = StContainerFenceRow { + allowed: true, + ..revoked.clone() + }; + assert!(matches!( + install_container_fence(&db, tx, &reopen), + Err(DeploymentError::FenceConflict) + )); + assert!(matches!( + install_container_fence(&db, tx, &first), + Err(DeploymentError::FenceConflict) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn captured_connection_authority_survives_replay_without_inferring_sender_authority() { + let db = TestDB::durable().unwrap(); + let self_sender = db.database_identity(); + let foreign_sender = Identity::ONE; + let self_connection = ConnectionId::from_u128(41); + let foreign_connection = ConnectionId::from_u128(42); + let ordinary_connection = ConnectionId::from_u128(43); + transact(&db, |tx| { + for (connection, sender, flags) in [ + (self_connection, self_sender, 1), + (foreign_connection, foreign_sender, 0), + ] { + tx.insert_st_client( + sender, + connection, + r#"{"iss":"platform","sub":"previously-admitted","exp":1}"#, + )?; + record_connection_auth(tx, connection, sender, flags)?; + } + tx.insert_st_client(self_sender, ordinary_connection, "ordinary JWT")?; + Ok(()) + }) + .unwrap(); + // TestDB::reopen expects zero connected clients. Reopen the same committed + // log explicitly to exercise crash recovery with outstanding connections. + let (db, durability, runtime, directory) = db.into_parts(); + let runtime = runtime.unwrap(); + let directory = directory.unwrap(); + let durability = durability.unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); + drop(durability); + let _runtime_guard = runtime.enter(); + let (db, durability) = TestDB::open_existing_durable( + &directory, + runtime.handle().clone(), + 0, + TestDB::DATABASE_IDENTITY, + TestDB::OWNER, + true, + ) + .unwrap(); + transact(&db, |tx| { + assert_eq!(connection_auth_flags(tx, self_connection, self_sender)?, 1); + assert_eq!(connection_auth_flags(tx, foreign_connection, foreign_sender)?, 0); + // Equal sender/database identities do not invent internal authority. + assert_eq!(connection_auth_flags(tx, ordinary_connection, self_sender)?, 0); + assert_eq!(tx.table_row_count(ST_CONNECTION_AUTH_ID), Some(2)); + assert!(matches!( + connection_auth_flags(tx, self_connection, foreign_sender), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); + db.clear_all_clients().unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); +} + +#[test] +fn connection_auth_and_client_rows_share_connect_and_cleanup_transactions() { + let db = TestDB::in_memory().unwrap(); + let sender = db.database_identity(); + let connection = ConnectionId::from_u128(77); + let rejected: Result<(), DeploymentError> = transact(&db, |tx| { + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(rejected.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_none()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 0); + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Ok(()) + }) + .unwrap(); + // A failed callback transaction cannot partially delete its captured auth. + let failed_callback: Result<(), DeploymentError> = transact(&db, |tx| { + tx.delete_st_client(sender, connection, db.database_identity())?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed_callback.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_some()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 1); + // Both successful callbacks and fallback cleanup use this deletion path. + tx.delete_st_client(sender, connection, db.database_identity())?; + Ok(()) + }) + .unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_installation_serializes_with_admitted_transactions() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(778u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 0, + target_set_hash: hash_bytes(b"self"), + allowed: true, + }; + transact(&db, |tx| install_container_fence(&db, tx, &first)).unwrap(); + let admitted = begin_mut_tx(&db); + check_container_fence(&admitted, source, 1, 0).unwrap(); + // A fence cannot commit midway through an already admitted transaction. + assert!(db + .try_begin_mut_tx( + spacetimedb_datastore::traits::IsolationLevel::Serializable, + Workload::ForTests + ) + .is_none()); + let _ = db.rollback_mut_tx(admitted); + transact(&db, |tx| { + install_container_fence(&db, tx, &StContainerFenceRow { generation: 2, ..first }) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 0), + Err(DeploymentError::ContainerFenced) + )); + check_container_fence(tx, source, 2, 0) + }) + .unwrap(); +} + +#[test] +fn same_revision_environment_publications_compare_the_last_committed_operation() { + use crate::db::environment; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use std::collections::BTreeMap; + use std::sync::{Arc, Barrier}; + + let db = TestDB::in_memory().unwrap(); + let limits = ContainerSpecLimits::default(); + let first = request(1, None); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), "initial".into())])) + .map_err(|error| DeploymentError::Database(DBError::Other(error.into())))?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let mut second = request(2, Some(&accepted)); + second.deployment = first.deployment.clone(); + let mut third = request(3, Some(&accepted)); + third.deployment = first.deployment.clone(); + let barrier = Arc::new(Barrier::new(3)); + let writers: Vec<_> = [(second.clone(), "second"), (third.clone(), "third")] + .into_iter() + .map(|(request, value)| { + let db = db.db.clone(); + let barrier = barrier.clone(); + let schema = schema.clone(); + std::thread::spawn(move || { + barrier.wait(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + check_deployment_commit(tx, &request, now(), &Default::default())?; + environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), value.into())])) + .map_err(|error| DeploymentError::Database(DBError::Other(error.into())))?; + record_deployment_commit(tx, &request, now(), &Default::default()) + }) + }) + }) + .collect(); + barrier.wait(); + // Join both physical writers before asserting, including when one failed. + let outcomes: Vec<_> = writers.into_iter().map(|writer| writer.join()).collect(); + let outcomes: Vec<_> = outcomes.into_iter().map(Result::unwrap).collect(); + assert_eq!(outcomes.iter().filter(|result| result.is_ok()).count(), 1); + let winner = outcomes.into_iter().find_map(Result::ok).unwrap(); + assert_eq!(winner.revision, accepted.revision); + assert_ne!(winner.operation_id, accepted.operation_id); + transact(&db, |tx| { + assert_eq!(current_publication(tx)?, Some(winner.clone())); + let expected = if winner.operation_id == second.operation_id { + "second" + } else { + "third" + }; + assert_eq!( + environment::snapshot(tx).map_err(|error| DeploymentError::Database(DBError::Other(error.into())))? + ["TOKEN"], + expected + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + // A later attempt has the current revision but an obsolete operation. + let mut stale = request(4, Some(&accepted)); + stale.deployment = first.deployment.clone(); + install_publication_fence(tx, stale.publication_epoch, stale.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &stale, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + abort_deployment_commit(tx, &stale), + Err(DeploymentError::RevisionConflict) + )); + // The exact current pair can close this attempt; the old pair cannot + // interpret its closed marker as proof that its precondition survived. + let mut current = stale.clone(); + current.expected_last_operation = Some(winner.operation_id); + abort_deployment_commit(tx, ¤t)?; + assert!(deployment_publication_aborted(tx, ¤t)?); + assert!(matches!( + deployment_publication_aborted(tx, &stale), + Err(DeploymentError::RevisionConflict) + )); + assert_eq!(current_publication(tx)?, Some(winner)); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn retained_receipt_binds_prior_operation_epoch_and_current_metadata() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &Default::default()) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = first.clone(); + changed.expected_last_operation = Some(request(9, None).operation_id); + assert!(matches!( + committed_deployment_operation(tx, &changed), + Err(DeploymentError::OperationConflict) + )); + changed = first.clone(); + changed.publication_epoch += 1; + assert!(matches!( + committed_deployment_operation(tx, &changed), + Err(DeploymentError::OperationConflict) + )); + assert_eq!(current_publication(tx)?, Some(accepted.clone())); + Ok(()) + }) + .unwrap(); + // A corrupted result epoch must fail even when its request hash still + // matches. Do not derive a replacement epoch from the publication fence. + transact(&db, |tx| { + let row = tx.iter(ST_DEPLOYMENT_OPERATION_ID)?.next().unwrap(); + let mut row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(mut receipt) = bsatn::from_slice(&row.commit_result).unwrap(); + receipt.result.publication_epoch = 0; + row.commit_result = bsatn::to_vec(&CommitReceipt::V1(receipt)).unwrap().into(); + tx.clear_table(ST_DEPLOYMENT_OPERATION_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_OPERATION_ID, &row)?; + assert!(matches!(current_publication(tx), Err(DeploymentError::CorruptMetadata))); + assert!(matches!( + committed_deployment_operation(tx, &first), + Err(DeploymentError::CorruptMetadata) + )); + tx.clear_table(ST_DEPLOYMENT_OPERATION_ID)?; + assert!(matches!(current_publication(tx), Err(DeploymentError::CorruptMetadata))); + let next = request(2, Some(&accepted)); + install_publication_fence(tx, next.publication_epoch, next.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &next, now(), &Default::default()), + Err(DeploymentError::CorruptMetadata) + )); + assert!(matches!( + abort_deployment_commit(tx, &next), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index a7117db5b71..88a553da487 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -26,6 +26,15 @@ pub mod update { pub use spacetimedb_engine::update::*; } +pub mod container_environment; +pub mod deployment; +pub mod hosted_admission { + pub use spacetimedb_engine::hosted_admission::*; +} + +#[cfg(test)] +mod retained_shutdown_tests; + /// Whether SpacetimeDB is run in memory, or persists objects and /// a message log to disk. #[derive(Clone, Copy)] diff --git a/crates/core/src/db/retained_shutdown_tests.rs b/crates/core/src/db/retained_shutdown_tests.rs new file mode 100644 index 00000000000..ed8217e2276 --- /dev/null +++ b/crates/core/src/db/retained_shutdown_tests.rs @@ -0,0 +1,64 @@ +use crate::db::{environment, relational_db::tests_utils::TestDB}; +use crate::error::DBError; +use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; +use spacetimedb_datastore::{execution_context::Workload, traits::IsolationLevel}; + +#[test] +fn operation_drain_retained_sql_and_subscription_handles_reject_after_writer_close() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.clone()); + db.shutdown().await; + let auth = spacetimedb_lib::identity::AuthCtx::new(db.owner_identity(), db.owner_identity()); + for statement in ["SELECT * FROM st_client", "SELECT value FROM st_env WHERE key = 'LATE'"] { + let error = crate::sql::execute::run( + db.clone(), + statement.into(), + auth.clone(), + Some(subscriptions.clone()), + None, + &mut vec![], + ) + .await + .err() + .unwrap(); + assert!(matches!(error, DBError::DatabaseClosed), "{error}"); + } + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Unsubscribe); + let schema = spacetimedb_lib::environment::EnvironmentSchema::new(vec![ + spacetimedb_lib::environment::EnvironmentDeclaration { + name: "LATE".into(), + constraint: spacetimedb_lib::environment::EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap(); + environment::replace( + &db, + &mut tx, + &schema, + &std::collections::BTreeMap::from([("LATE".into(), "denied".into())]), + ) + .unwrap(); + let event = ModuleEvent { + timestamp: spacetimedb_lib::Timestamp::now(), + caller_identity: db.owner_identity(), + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + execution_budget_used: spacetimedb_client_api_messages::energy::FunctionBudget::ZERO, + host_execution_duration: std::time::Duration::ZERO, + request_id: None, + timer: None, + }; + let error = subscriptions.commit_and_broadcast_event(None, event, tx).err().unwrap(); + assert!(matches!(error, DBError::DatabaseClosed)); + let tx = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&tx, "LATE").unwrap(), None); + let _ = db.release_tx(tx); + }); +} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 0f8d5621b3f..7417927fe7f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -34,6 +34,8 @@ pub enum NodesError { InvalidEnvironmentKey, #[error("too many outstanding byte sources for environment read")] EnvironmentSourceLimit, + #[error("hosted invocation rejected: {0}")] + HostedInvocationRejected(String), #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/container_environment.rs b/crates/core/src/host/container_environment.rs new file mode 100644 index 00000000000..45535df1a09 --- /dev/null +++ b/crates/core/src/host/container_environment.rs @@ -0,0 +1,188 @@ +//! Bounded, durable host operations for immutable container environments. +//! +//! The caller must confirm current control authority and authoritative leader +//! before calling. These local host APIs are not an external authentication +//! interface. Historical restore must keep admission closed until current +//! operational fences have been reconciled. No in-memory production fallback. + +use crate::db::container_environment::{ + self as storage, EnvironmentClosedReceipt, EnvironmentSnapshotError, SecretEnvironment, +}; +use crate::db::relational_db::{MutTx, RelationalDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +/// Bounds queued/blocked transactions even when the async caller is cancelled. +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +/// The barrier is specific to this proof. It can advance on an exact retry. +#[derive(Debug)] +pub struct Durable { + pub receipt: T, + pub durable_through: u64, +} + +pub type DurableSnapshotReceipt = Durable; +pub type DurableEnvironmentValues = Durable; +pub type DurableClosedReceipt = Durable; + +pub async fn capture( + db: Arc, + scope: EnvironmentSnapshotScope, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| storage::capture(&action_db, tx, &scope)).await +} + +pub async fn read( + db: Arc, + receipt: EnvironmentSnapshotReceipt, +) -> Result { + read_with_capacity(db, receipt, OPERATIONS.clone()).await +} + +async fn read_with_capacity( + db: Arc, + receipt: EnvironmentSnapshotReceipt, + capacity: Arc, +) -> Result { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let result = storage::read(&action_db, &tx, &receipt); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + result.map(|result| (offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +pub async fn close( + db: Arc, + scope: EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| { + storage::close(&action_db, tx, &scope, closed_through_generation) + }) + .await +} + +async fn mutate( + db: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + mutate_with_capacity(db, OPERATIONS.clone(), action).await +} + +async fn mutate_with_capacity( + db: Arc, + capacity: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + // A cancelled waiter drops only this JoinHandle. The actual operation + // keeps its database and finite slot until commit and durability finish. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, result) = action_db.with_auto_rollback(tx, action)?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .ok_or(EnvironmentSnapshotError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, EnvironmentSnapshotError>((offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +#[cfg(test)] +#[path = "container_environment/durability_tests.rs"] +mod durability_tests; + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use std::time::Duration; + + #[test] + fn container_environment_cancelled_transaction_retains_capacity_until_worker_finishes() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release, release_rx) = std::sync::mpsc::sync_channel(1); + let blocked = db + .runtime() + .unwrap() + .spawn(mutate_with_capacity(db.db.clone(), capacity.clone(), move |_| { + entered.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Err::<(), _>(EnvironmentSnapshotError::InvalidEnvironment) + })); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + blocked.abort(); + db.runtime().unwrap().block_on(async { + assert!(blocked.await.unwrap_err().is_cancelled()); + let retry = mutate_with_capacity(db.db.clone(), capacity.clone(), |_| Ok(())).await; + assert!(matches!(retry, Err(EnvironmentSnapshotError::Capacity))); + }); + release.send(()).unwrap(); + db.runtime().unwrap().block_on(async { + // This waits for the actual cancelled caller's blocking worker to + // return, not merely for cancellation of its async JoinHandle. + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + mutate_with_capacity(db.db.clone(), capacity, |_| Ok(())).await.unwrap(); + }); + } +} diff --git a/crates/core/src/host/container_environment/durability_tests.rs b/crates/core/src/host/container_environment/durability_tests.rs new file mode 100644 index 00000000000..e2e533c78db --- /dev/null +++ b/crates/core/src/host/container_environment/durability_tests.rs @@ -0,0 +1,313 @@ +//! Real local writes with only their durability acknowledgment delayed. +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::{ + local_durability, + tests_utils::{TempReplicaDir, TestDB}, + LocalDurability, +}; +use crate::db::{environment, persistence::Persistence}; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_durability::{Close, Durability, DurableOffset, PreparedTx}; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent}; +use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use spacetimedb_lib::{hash_bytes, Timestamp, Uuid}; +use std::time::Duration; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn environment_schema(keys: &[String]) -> EnvironmentSchema { + EnvironmentSchema::new( + keys.iter() + .map(|name| EnvironmentDeclaration { + name: name.clone(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }) + .collect(), + ) + .unwrap() +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let schema = environment_schema(&spec.env_keys); + let generated = spacetimedb_lib::deployment::system_empty::generate(&schema).unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + expected_last_operation: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(generated.descriptor), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + environment::replace( + db, + tx, + &schema, + &spec.env_keys.iter().map(|key| (key.clone(), "before".into())).collect(), + )?; + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + publication_operation: request.operation_id, + publication_epoch: request.publication_epoch, + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +struct DelayedAcknowledgment { + writer: LocalDurability, + acknowledged: DurableOffset, +} +impl Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> Close { + self.writer.close() + } +} + +struct Fixture { + db: Arc, + writer: LocalDurability, + acknowledge: tokio::sync::watch::Sender>, + _directory: TempReplicaDir, +} +impl Fixture { + async fn new() -> Self { + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = + local_durability((*directory).clone(), spacetimedb_runtime::Handle::tokio_current(), None) + .await + .unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: spacetimedb_runtime::Handle::tokio_current(), + }), + None, + 0, + ) + .unwrap(), + ); + Self { + db, + writer, + acknowledge, + _directory: directory, + } + } + async fn acknowledge_current(&self) -> u64 { + let tx = self.db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = self.db.release_tx(tx); + self.db.report_read_tx_metrics(reducer, metrics); + let mut actual = self.writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + self.acknowledge.send_replace(Some(durable)); + offset + } + async fn finish(self) { + self.db.shutdown().await; + drop(self.db); + self.writer.close().await; + } +} + +async fn wait_for_slot(capacity: &Arc) { + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn released(capacity: &Arc) { + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_capture_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (captured, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::capture(&action_db, tx, &scope)?; + captured.send(receipt.clone()).unwrap(); + Ok(receipt) + })); + let captured = tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + // This read waits for the actual capture transaction to commit. + db.with_read_only(Workload::ForTests, |tx| storage::read(&db, tx, &captured)) + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read(db.clone(), captured).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_read_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(read_with_capacity(db.clone(), receipt.clone(), capacity.clone())); + wait_for_slot(&capacity).await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + read_with_capacity(db.clone(), receipt.clone(), capacity.clone()).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read_with_capacity(db.clone(), receipt, capacity).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_close_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &self_fence(&db, 2, false)) + }) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (closed, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::close(&action_db, tx, &scope, 2)?; + closed.send(()).unwrap(); + Ok(receipt) + })); + tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + assert_eq!( + tx.table_row_count(spacetimedb_datastore::system_tables::ST_CONTAINER_ENVIRONMENT_ID), + Some(0) + ); + }); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + assert!(matches!( + read(db.clone(), receipt).await, + Err(EnvironmentSnapshotError::Fenced) + )); + drop(db); + fixture.finish().await; +} diff --git a/crates/core/src/host/container_fence.rs b/crates/core/src/host/container_fence.rs new file mode 100644 index 00000000000..4b872fbcea1 --- /dev/null +++ b/crates/core/src/host/container_fence.rs @@ -0,0 +1,204 @@ +//! Bounded host-only inspection and durable denial of retained source fences. +//! +//! The authenticated platform coordinator owns control membership checks and +//! actor drainage. These APIs do not authorize an external client or expose the +//! protected fence table through SQL, subscriptions, or module syscalls. + +use crate::db::deployment::{deny_container_fence, DeploymentError, FenceDenial}; +use crate::db::relational_db::RelationalDB; +use crate::host::container_environment::Durable; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::ScanOrIndex; +use spacetimedb_datastore::system_tables::{StContainerFenceRow, ST_CONTAINER_FENCE_ID}; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::Identity; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::ops::Bound; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +pub const FENCE_PAGE_SIZE: usize = 64; +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +#[derive(Debug, thiserror::Error)] +pub enum FenceOperationError { + #[error("receiving host fence operation capacity exhausted")] + Capacity, + #[error("receiving host fence storage is unavailable")] + Storage, + #[error("receiving host fence durability is unavailable")] + DurabilityUnavailable, + #[error("receiving host fence durability failed")] + DurabilityFailed, + #[error("receiving host fence index is unavailable")] + IndexUnavailable, + #[error("receiving host fence inventory changed")] + RevisionChanged, + #[error(transparent)] + Deployment(#[from] DeploymentError), +} + +#[derive(Debug)] +pub struct FencePage { + /// Includes denied rows so every bounded physical page advances its cursor. + pub rows: Vec, + /// Last source in this page, or the input cursor if the page is empty. + pub next_source: Option, + pub complete: bool, + /// Read under the same database transaction as the rows. Restart the scan + /// if this differs from the previous page or a known local denial result. + pub fence_revision: u64, +} + +/// Read at most 64 rows using the protected source Identity B-tree. We refuse +/// the datastore's scan fallback: row order and physical work must be bounded. +pub async fn page(db: Arc, after: Option) -> Result { + let permit = OPERATIONS + .clone() + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let tx = db.begin_tx(Workload::Internal); + let result = (|| { + let lower = after.map_or(Bound::Unbounded, |identity| { + Bound::Excluded(AlgebraicValue::U256(identity.to_u256().into())) + }); + let range = db + .iter_by_col_range(&tx, ST_CONTAINER_FENCE_ID, ColId(0), (lower, Bound::Unbounded)) + .map_err(|_| FenceOperationError::Storage)?; + let ScanOrIndex::Index(mut range) = range else { + return Err(FenceOperationError::IndexUnavailable); + }; + let rows = range + .by_ref() + .take(FENCE_PAGE_SIZE) + .map(|row| StContainerFenceRow::try_from(row).map_err(|_| FenceOperationError::Storage)) + .collect::, _>>()?; + let complete = range.next().is_none(); + let next_source = rows.last().map(|row| row.source_identity.into()).or(after); + Ok(FencePage { + rows, + next_source, + complete, + fence_revision: db.hosted_admission().fence_revision(), + }) + })(); + let (_, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + result + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +/// The coordinator must have excluded this exact source from current control +/// authority. For a check that must serialize with an inventory lock, use the +/// synchronous `deployment::deny_container_fence` inside the caller's own +/// serializable transaction instead. Neither API performs that control check. +pub async fn deny_orphan( + db: Arc, + expected: StContainerFenceRow, +) -> Result, FenceOperationError> { + deny_with_capacity(db, expected, OPERATIONS.clone()).await +} + +/// Confirm durability of all fences visible at one exact physical revision. +/// In particular, a retry that sees a previous owner's committed denial must +/// still wait for that denial's storage acknowledgment. The coordinator must +/// recheck the revision under its final database transaction before admission; +/// this receipt does not freeze future mutations or confer control authority. +pub async fn confirm_revision( + db: Arc, + expected_physical: u64, +) -> Result, FenceOperationError> { + confirm_with_capacity(db, expected_physical, OPERATIONS.clone()).await +} + +async fn confirm_with_capacity( + db: Arc, + expected_physical: u64, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let durable_through = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let physical = action_db.hosted_admission().fence_revision(); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + if physical != expected_physical || physical == u64::MAX { + return Err(FenceOperationError::RevisionChanged); + } + Ok(offset) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt: (), + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +async fn deny_with_capacity( + db: Arc, + expected: StContainerFenceRow, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + // A caller's cancellation drops only this JoinHandle. The owned task keeps + // both the permit and database alive through physical commit and durability. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, receipt) = + action_db.with_auto_rollback(tx, |tx| deny_container_fence(&action_db, tx, &expected))?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| FenceOperationError::Storage)? + .ok_or(FenceOperationError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, FenceOperationError>((offset, receipt)) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + // Do not drop the physical database before the owned durability wait. + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/container_fence/tests.rs b/crates/core/src/host/container_fence/tests.rs new file mode 100644 index 00000000000..41e203a072a --- /dev/null +++ b/crates/core/src/host/container_fence/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::db::deployment::{check_container_fence, install_container_fence}; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_lib::hash_bytes; +use std::time::Duration; + +fn fence(source: u64) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: Identity::from_u256(source.into()).into(), + generation: 7, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"retained target set"), + allowed: true, + } +} + +fn install(db: &RelationalDB, row: &StContainerFenceRow) { + db.with_auto_commit(Workload::ForTests, |tx| install_container_fence(db, tx, row)) + .unwrap(); +} + +#[tokio::test] +async fn container_fence_pages_follow_identity_index_and_include_denied_rows() { + let db = TestDB::in_memory().unwrap(); + // Reverse insertion order deliberately differs from source index order. + for source in (0..130).rev() { + install( + &db, + &StContainerFenceRow { + allowed: source % 3 == 0, + ..fence(source) + }, + ); + } + let revision = db.hosted_admission().fence_revision(); + let mut after = None; + let mut identities = Vec::new(); + for expected_size in [64, 64, 2] { + let page = page(db.db.clone(), after).await.unwrap(); + assert_eq!(page.rows.len(), expected_size); + assert_eq!(page.complete, expected_size == 2); + assert_eq!(page.fence_revision, revision); + identities.extend(page.rows.iter().map(|row| Identity::from(row.source_identity))); + after = page.next_source; + assert_eq!(after, identities.last().copied()); + } + assert_eq!( + identities, + (0..130u64).map(|n| Identity::from_u256(n.into())).collect::>() + ); + let empty = page(db.db.clone(), after).await.unwrap(); + assert!(empty.rows.is_empty() && empty.complete); + assert_eq!(empty.next_source, after); +} + +#[tokio::test] +async fn container_fence_missing_index_refuses_unbounded_scan_and_memory_refuses_denial() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(17); + install(&db, &expected); + assert!(matches!( + deny_orphan(db.db.clone(), expected).await, + Err(FenceOperationError::DurabilityUnavailable) + )); + db.with_auto_commit(Workload::ForTests, |tx| { + db.drop_index(tx, spacetimedb_primitives::IndexId(34)) + }) + .unwrap(); + assert!(matches!( + page(db.db.clone(), None).await, + Err(FenceOperationError::IndexUnavailable) + )); +} + +#[test] +fn container_fence_orphan_denial_is_durable_exact_and_cannot_reopen_generation() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let expected = fence(19); + install(&db, &expected); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let before = db.hosted_admission().fence_revision(); + let first = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(first.receipt.revision_before, before); + assert_eq!(first.receipt.revision_after, before + 1); + assert!(!db.hosted_admission().is_open()); + assert_eq!( + first.receipt.row, + StContainerFenceRow { + allowed: false, + ..expected.clone() + } + ); + assert!(db.durable_tx_offset().unwrap().get().unwrap().unwrap() >= first.durable_through); + let retry = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(retry.receipt.revision_before, before + 1); + assert_eq!(retry.receipt.revision_after, before + 1); + let db = db.reopen().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision + ), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + install_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + let retry = deny_container_fence(&db, tx, &expected)?; + assert_eq!(retry.row, first.receipt.row); + assert_eq!(retry.revision_before, retry.revision_after); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install( + &db, + &StContainerFenceRow { + generation: expected.generation + 1, + ..expected + }, + ); +} + +#[test] +fn container_fence_orphan_cas_rejects_missing_changed_and_invalid_observations() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(20); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + deny_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install(&db, &expected); + let next = StContainerFenceRow { + generation: expected.generation + 1, + ..expected.clone() + }; + install(&db, &next); + let revision = db.hosted_admission().fence_revision(); + db.with_auto_commit(Workload::ForTests, |tx| { + for stale in [ + expected, + StContainerFenceRow { + allowed: false, + ..next.clone() + }, + StContainerFenceRow { + target_set_hash: hash_bytes(b"different"), + ..next.clone() + }, + StContainerFenceRow { + target_grant_revision: 4, + ..next.clone() + }, + ] { + assert!(matches!( + deny_container_fence(&db, tx, &stale), + Err(DeploymentError::FenceConflict) + )); + } + check_container_fence( + tx, + next.source_identity.into(), + next.generation, + next.target_grant_revision, + )?; + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + assert_eq!(db.hosted_admission().fence_revision(), revision); +} + +#[tokio::test] +async fn container_fence_physical_change_behind_cursor_invalidates_completion() { + let db = TestDB::in_memory().unwrap(); + install(&db, &fence(100)); + let ticket = db.hosted_admission().begin().unwrap(); + let first = page(db.db.clone(), None).await.unwrap(); + assert!(first.complete); + install(&db, &fence(1)); + assert!(ticket.complete_with_fence_revision(first.fence_revision).is_err()); + assert!(!db.hosted_admission().is_open()); + let restarted = page(db.db.clone(), None).await.unwrap(); + assert_eq!(restarted.rows.len(), 2); + assert_ne!(first.fence_revision, restarted.fence_revision); +} + +#[test] +fn container_fence_orphan_rollback_keeps_row_and_invalidates_physical_scan() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(21); + install(&db, &expected); + let before = db.hosted_admission().fence_revision(); + let ticket = db.hosted_admission().begin().unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| { + deny_container_fence(&db, tx, &expected)?; + Err::<(), _>(DeploymentError::CorruptMetadata) + }); + assert!(result.is_err()); + assert!(ticket.complete_with_fence_revision(before).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| { + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision, + ) + }) + .unwrap(); +} + +/// Real local storage with its durable acknowledgment deliberately withheld. +/// This models a lagging quorum without replacing the transaction or writer. +struct DelayedAcknowledgment { + writer: Arc>, + acknowledged: spacetimedb_durability::DurableOffset, +} + +impl spacetimedb_durability::Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: spacetimedb_durability::PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> spacetimedb_durability::DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> spacetimedb_durability::Close { + self.writer.close() + } +} + +async fn delayed_storage() -> ( + crate::db::relational_db::tests_utils::TempReplicaDir, + Arc, + crate::db::relational_db::LocalDurability, + tokio::sync::watch::Sender>, +) { + use crate::db::persistence::Persistence; + use crate::db::relational_db::{local_durability, tests_utils::TempReplicaDir}; + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = + local_durability((*directory).clone(), spacetimedb_runtime::Handle::tokio_current(), None) + .await + .unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: spacetimedb_runtime::Handle::tokio_current(), + }), + None, + 0, + ) + .unwrap(), + ); + (directory, db, writer, acknowledge) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_cancelled_denial_retains_capacity_through_actual_durability_wait() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(22); + install(&db, &expected); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(deny_with_capacity(db.clone(), expected.clone(), capacity.clone())); + let row = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let page = page(db.clone(), None).await.unwrap(); + if let Some(row) = page.rows.first().filter(|row| !row.allowed) { + break row.clone(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!row.allowed); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + deny_with_capacity(db.clone(), expected.clone(), capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + // The detached owner finished only once its physical acknowledgment arrived. + db.shutdown().await; + drop(db); + writer.close().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_retry_barrier_waits_for_prior_commit_and_retains_cancelled_capacity() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(23); + install(&db, &expected); + // A previous coordinator has committed but never confirmed durability. + let denied = db + .with_auto_commit(Workload::ForTests, |tx| deny_container_fence(&db, tx, &expected)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + // Changed inventory fails immediately, even while acknowledgments lag. + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(5), + confirm_with_capacity(db.clone(), denied.revision_before, capacity.clone()) + ) + .await + .unwrap(), + Err(FenceOperationError::RevisionChanged) + )); + let caller = tokio::spawn(confirm_with_capacity( + db.clone(), + denied.revision_after, + capacity.clone(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!caller.is_finished()); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert!(matches!( + confirm_with_capacity(db.clone(), denied.revision_after, capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + let proof = confirm_revision(db.clone(), denied.revision_after).await.unwrap(); + assert_eq!(proof.durable_through, offset); + // The proof is only a barrier, so finalization still checks current revision. + install(&db, &fence(24)); + assert!(matches!( + confirm_revision(db.clone(), denied.revision_after).await, + Err(FenceOperationError::RevisionChanged) + )); + db.shutdown().await; + drop(db); + writer.close().await; +} diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs new file mode 100644 index 00000000000..1c267628529 --- /dev/null +++ b/crates/core/src/host/empty_module.rs @@ -0,0 +1,42 @@ +//! Versioned built-in module for a database published with only a container. +//! +//! This real Wasm program declares container configuration's environment schema +//! and hosted_auth_v1, with no user tables or callable functions. Database +//! initialization, system tables, subscriptions, and migration use the normal +//! host. Verification compares exact generated bytes before trusting its origin. + +use spacetimedb_datastore::{system_tables::ModuleKind, traits::Program}; +use spacetimedb_lib::deployment::{system_empty, SystemEmptyModule}; +use spacetimedb_lib::environment::EnvironmentSchema; + +pub const VERSION: u32 = system_empty::VERSION; + +/// Return the exact bundled program for a recognized system module version. +/// Unknown versions fail closed instead of silently selecting the latest one. +pub fn program(version: u32) -> Option { + (version == VERSION).then(|| Program { + hash: system_empty::empty().descriptor.program_hash, + bytes: system_empty::empty().bytes.clone(), + kind: ModuleKind::WASM, + }) +} + +/// Validate a system-empty deployment against immutable platform bytes, not +/// against an empty-looking schema supplied by a publisher or a claimed hash. +pub fn matches_program(descriptor: &SystemEmptyModule, candidate: &Program) -> bool { + candidate.kind == ModuleKind::WASM + && candidate.hash == descriptor.program_hash + && system_empty::verify(descriptor, &candidate.bytes).is_ok() +} + +pub fn declared_program(environment: &EnvironmentSchema) -> Result { + let generated = system_empty::generate(environment)?; + Ok(Program { + hash: generated.descriptor.program_hash, + bytes: generated.bytes, + kind: ModuleKind::WASM, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/empty_module/README.md b/crates/core/src/host/empty_module/README.md new file mode 100644 index 00000000000..5422f8b314e --- /dev/null +++ b/crates/core/src/host/empty_module/README.md @@ -0,0 +1,9 @@ +# Container-only platform module + +Version 2 is generated in `spacetimedb-lib::deployment::system_empty` from the validated declarations in container configuration. It exports the required V10 ABI entry points and declares ENV section15 and capability section16. It has no user tables, reducers, procedures, views, or HTTP handlers. + +Declarations are normalized before generation. Their exact bytes determine the Wasm program hash; changing declarations therefore selects a new module. The generated memory has equal minimum and maximum page counts, sized to contain the complete bounded metadata, including schemas larger than one Wasm page. + +The shared verifier reads only bounded declaration data, regenerates the platform program, and compares every byte and the claimed program identity. It never executes the supplied program to decide whether it is platform code. The unshipped version1 prototype is not accepted. + +Run the canonical generator and protocol tests with `cargo test -p spacetimedb-lib --lib deployment::`. The actual host extraction and initialization tests are in `crates/core/src/host/empty_module/tests.rs`. diff --git a/crates/core/src/host/empty_module/tests.rs b/crates/core/src/host/empty_module/tests.rs new file mode 100644 index 00000000000..b509d411ef7 --- /dev/null +++ b/crates/core/src/host/empty_module/tests.rs @@ -0,0 +1,128 @@ +use super::*; +use crate::{host::extract_schema, messages::control_db::HostType}; +use spacetimedb_lib::{hash_bytes, Hash}; +use spacetimedb_schema::def::RawModuleDefVersion; + +#[test] +fn recognition_requires_exact_version_kind_hash_and_bytes() { + assert!(program(0).is_none()); + assert!(program(1).is_none()); + let descriptor = system_empty::empty().descriptor; + let mut candidate = program(VERSION).unwrap(); + assert!(matches_program(&descriptor, &candidate)); + candidate.kind = ModuleKind::JS; + assert!(!matches_program(&descriptor, &candidate)); + candidate.kind = ModuleKind::WASM; + candidate.hash = Hash::ZERO; + assert!(!matches_program(&descriptor, &candidate)); + candidate = program(VERSION).unwrap(); + candidate.bytes[0] ^= 1; + candidate.hash = hash_bytes(&candidate.bytes); + assert!(!matches_program(&descriptor, &candidate)); + assert!(!matches_program(&descriptor, &Program::empty(ModuleKind::WASM))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn actual_wasm_host_reads_empty_and_large_declared_environment_schemas() { + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration}; + let declared = EnvironmentSchema::new( + (0..16) + .map(|index| EnvironmentDeclaration { + name: format!("KEY_{index}"), + constraint: EnvironmentConstraint::Literal("x".repeat(8192)), + optional: true, + }) + .collect(), + ) + .unwrap(); + for environment in [EnvironmentSchema::default(), declared] { + let program = declared_program(&environment).unwrap(); + let descriptor = SystemEmptyModule { + version: VERSION, + program_hash: program.hash, + }; + assert!(matches_program(&descriptor, &program)); + let module = extract_schema(program.bytes, HostType::Wasm).await.unwrap(); + assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V10); + assert!(module.supports_hosted_auth_v1()); + assert!(module.environment_declared()); + assert_eq!(module.environment(), &environment); + assert!(module.tables().next().is_none()); + assert!(module.reducers().next().is_none()); + assert!(module.procedures().next().is_none()); + assert!(module.views().next().is_none()); + assert!(module.typespace().types.is_empty()); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn actual_host_initializes_a_database_with_the_bundled_program() { + use crate::{ + db::{persistence::LocalPersistenceProvider, Config, Storage}, + energy::NullEnergyMonitor, + host::{FunctionArgs, HostController, HostRuntimeConfig, ProgramStorage}, + messages::control_db::Database, + util::jobs::JobCores, + }; + use spacetimedb_lib::Identity; + use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; + use std::{sync::Arc, time::Duration}; + + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let storage: ProgramStorage = Arc::new(|hash| async move { + Ok(program(VERSION) + .filter(|program| program.hash == hash) + .map(|program| program.bytes)) + }); + let controller = HostController::new( + data_dir.clone(), + Config { + storage: Storage::Memory, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage, + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id: 0xe001, + database_identity: Identity::from_u256(0xe001_u32.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: system_empty::empty().descriptor.program_hash, + bootstrap_generation: 0, + }; + let module = controller + .get_or_launch_module_host(database.clone(), 0xe001) + .await + .unwrap(); + let stored = module + .relational_db() + .program() + .unwrap() + .expect("host initialization must persist the actual program"); + assert!(matches_program(&system_empty::empty().descriptor, &stored)); + let metadata = module + .relational_db() + .metadata() + .unwrap() + .expect("database must be initialized"); + assert_eq!(metadata.program_hash, system_empty::empty().descriptor.program_hash); + assert_eq!(metadata.database_identity, database.database_identity); + assert_eq!(metadata.owner_identity, database.owner_identity); + assert!(module.info.module_def.supports_hosted_auth_v1()); + assert!(module.info.module_def.tables().next().is_none()); + assert!(module + .call_reducer(Identity::ONE, None, None, None, None, "init", FunctionArgs::Nullary) + .await + .is_err()); + drop(module); + controller + .exit_module_host(0xe001, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index b6bb8b80ee0..b74b94f7a9c 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -6,6 +6,7 @@ use super::{Scheduler, UpdateDatabaseResult}; use crate::client::{ClientActorId, ClientName}; use crate::config::{ModuleHttpConfig, V8Config, WasmConfig}; use crate::database_logger::DatabaseLogger; +use crate::db::deployment::DeploymentCommit; use crate::db::persistence::PersistenceProvider; use crate::db::relational_db::{self, spawn_view_cleanup_loop, DiskSizeFn, RelationalDB, Txdata}; use crate::db::{self, spawn_tx_metrics_recorder}; @@ -25,15 +26,17 @@ use crate::worker_metrics::{ record_module_host_init_attempt, record_module_host_init_failure, record_module_host_unexpected_exit, ModuleHostInitFailureCause, WORKER_METRICS, }; -use anyhow::{bail, Context}; +use anyhow::{anyhow, bail, Context}; use async_trait::async_trait; use durability::{Durability, EmptyHistory}; +use futures::FutureExt as _; use log::{info, trace, warn}; +#[cfg(test)] use parking_lot::Mutex; use scopeguard::{defer, guard}; use spacetimedb_commitlog::SizeOnDisk; use spacetimedb_data_structures::error_stream::ErrorStream; -use spacetimedb_data_structures::map::{IntMap, IntSet}; +use spacetimedb_data_structures::map::IntSet; use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; @@ -51,12 +54,8 @@ use std::future::Future; use std::ops::Deref; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; -use tokio::time::error::Elapsed; -use tokio::time::{interval_at, timeout, Instant}; - -#[cfg(test)] -mod invocation_flags_tests; +use tokio::sync::{watch, RwLock as AsyncRwLock}; +use tokio::time::{timeout, Instant}; // TODO: // @@ -73,14 +72,36 @@ const IN_MEMORY_DATABASE_LOGGER_MAX_SIZE: u64 = 0x1_000_000; /// A shared mutable cell containing a module host and associated database. type HostCell = Arc>>; -/// The registry of all running hosts. -type Hosts = Arc>>; +mod lifecycle; +mod registry; +use registry::{Hosts, Registration}; + +#[cfg(test)] +mod lifecycle_tests; + +#[cfg(test)] +mod deployment_tests; + +#[cfg(test)] +mod execution_deadline_tests; + +#[cfg(test)] +static FAIL_NEXT_DEPLOYMENT_ACTIVATION: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); pub type ExternalDurability = (Arc>, DiskSizeFn); #[async_trait] pub trait ExternalStorage: Send + Sync + 'static { async fn lookup(&self, program_hash: Hash) -> anyhow::Result>>; + + /// Resolve the authorized initial deployment from the same control + /// transaction that created the database. A failed lookup must return an + /// error, never None: None is reserved for legacy module-only databases. + /// Called only when no initialized program exists in the user database. + async fn initial_deployment(&self, _database: &Database) -> anyhow::Result> { + Ok(None) + } } #[async_trait] impl ExternalStorage for F @@ -506,14 +527,14 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let rx = tokio::spawn(async move { - let initialized = this.try_init_host(database, replica_id).await?; + let initialized = this.try_init_host(database, replica_id, &guard).await?; let HostInit { host, bootstrap_completion, } = initialized; let rx = host.module.subscribe(); - *guard = Some(host); + guard.install(host); Ok::<_, anyhow::Error>((rx, bootstrap_completion)) }) @@ -591,18 +612,22 @@ impl HostController { program_bytes: Box<[u8]>, policy: MigrationPolicy, ) -> anyhow::Result { - self.update_module_host_with_environment( + self.update_module_host_with_environment_and_deployment( database, host_type, replica_id, program_bytes, policy, Default::default(), + None, ) .await } + /// Publish the complete environment with the module. No deployment action + /// is implied by this entry point. #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] pub async fn update_module_host_with_environment( &self, database: Database, @@ -611,6 +636,56 @@ impl HostController { program_bytes: Box<[u8]>, policy: MigrationPolicy, environment: std::collections::BTreeMap, + ) -> anyhow::Result { + self.update_module_host_with_environment_and_deployment( + database, + host_type, + replica_id, + program_bytes, + policy, + environment, + None, + ) + .await + } + + /// Authorized coordinator entry point with an omitted, empty environment. + /// Deployment and operation receipt commit in the migration transaction. + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_deployment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: Option, + ) -> anyhow::Result { + self.update_module_host_with_environment_and_deployment( + database, + host_type, + replica_id, + program_bytes, + policy, + Default::default(), + deployment, + ) + .await + } + + /// Publish the complete environment and authorized deployment in the same + /// module migration transaction, retaining ownership through activation. + #[tracing::instrument(level = "trace", skip_all, err)] + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_environment_and_deployment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -648,28 +723,57 @@ impl HostController { let mut host = match guard.take() { None => { trace!("host not running, try_init"); - this.try_init_host(database, replica_id).await?.host + this.try_init_host(database, replica_id, &guard).await?.host } Some(host) => { trace!("host found, updating"); host } }; - let update_result = host - .update_module( - this.runtimes.clone(), - program, - policy, - this.energy_monitor.clone(), - this.unregister_fn(replica_id, database_identity), - this.db_cores.take(), - environment, - ) - .await; + let mut database_committed = false; + let registration = guard.registration(); + let update_result = std::panic::AssertUnwindSafe(host.update_module( + this.runtimes.clone(), + program, + policy, + environment, + deployment, + this.energy_monitor.clone(), + this.unregister_fn(registration.clone(), database_identity), + registration, + this.db_cores.take(), + &mut database_committed, + )) + .catch_unwind() + .await; + let update_result = match update_result { + Ok(result) => result, + Err(panic) => { + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + std::panic::resume_unwind(panic); + } + }; + + if update_result.is_err() && database_committed { + // Schema/program/receipt already committed. The previous + // executable cannot be retained after activation failure. + // Close clients/scheduler and reconstruct from stored program + // on the next leader lookup or reconciliation attempt. + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + } else { + guard.install(host); + } - // Rejected publication leaves the existing host usable. Restore it - // before propagating validation or migration failure to the caller. - *guard = Some(host); update_result }) .await??; @@ -715,66 +819,28 @@ impl HostController { /// Release all resources of the [`ModuleHost`] identified by `replica_id`, /// and deregister it from the controller. + /// + /// A timeout returns an error while the owned close continues. Only success + /// confirms the configured storage writer has completed shutdown. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { - let start = Instant::now(); - if tokio::time::timeout(timeout, self.exit_module_host_and_join(replica_id)) + tokio::time::timeout(timeout, self.exit_module_host_and_join(replica_id)) .await - .is_err() - { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); - } - Ok(()) + .map_err(|_| anyhow!("replica {replica_id} shutdown is still pending after {timeout:?}"))? } - /// Wait for actual module and database closure, without treating an elapsed - /// request deadline as completion. The caller must retain this future and - /// exclude new launch admission until it returns, including if its own - /// request waiter is cancelled. + /// Join the canonical owned close without a waiter deadline. Cancellation + /// leaves the close owner and its registry pin active; later callers join + /// the same completion before launch admission can reopen the replica. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host_and_join(&self, replica_id: u64) -> Result<(), anyhow::Error> { - let Some(lock) = self.hosts.lock().remove(&replica_id) else { - return Ok(()); - }; - // To debug the potential deadlock issue reported in - // https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - // we'll log a warning every 5s if we can't acquire an exclusive lock. - let start = Instant::now(); - let mut t = interval_at(start + Duration::from_secs(5), Duration::from_secs(5)); - let warn_blocked = tokio::spawn(async move { - loop { - t.tick().await; - warn!( - "blocked waiting to exit module for replica {} since {}s", - replica_id, - start.elapsed().as_secs_f32() - ); - } - }); - defer!(warn_blocked.abort()); - - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { + let Some(request) = registry::close(&self.hosts, replica_id) else { return Ok(()); }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); - - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); - - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - Ok(()) + if let Some(owner) = request.owner { + tokio::spawn(owner.run()); + } + request.completion.wait().await } /// Get the [`ModuleHost`] identified by `replica_id` or return an error @@ -831,45 +897,47 @@ impl HostController { self.hosts.lock().keys().copied().collect() } - /// On-panic callback passed to [`ModuleHost`]s created by this controller. - /// - /// Removes the module with the given `replica_id` from this controller. - fn unregister_fn(&self, replica_id: u64, database_identity: Identity) -> impl Fn() + Send + Sync + 'static + use<> { - let hosts = Arc::downgrade(&self.hosts); + /// On-panic callbacks are scoped to one installed executable and cell. + /// A stale callback cannot unregister an updated or reopened database. + fn unregister_fn( + &self, + registration: Registration, + database_identity: Identity, + ) -> impl Fn() + Send + Sync + 'static + use<> { + let runtime = tokio::runtime::Handle::current(); move || { - let unregistered = hosts - .upgrade() - .is_some_and(|hosts| hosts.lock().remove(&replica_id).is_some()); - if unregistered { + if let Some(request) = registration.close_if_current() + && let Some(owner) = request.owner + { record_module_host_unexpected_exit(database_identity); + runtime.spawn(owner.run()); } } } - /// Acquire a write lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_write_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.write_owned()).await + /// The total wait includes any ongoing close and cell reacquisition. + async fn acquire_write_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::write(&self.hosts, replica_id)).await? } - /// Acquire a read lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_read_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.read_owned()).await + async fn acquire_read_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::read(&self.hosts, replica_id)).await? } - async fn try_init_host(&self, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init_host( + &self, + database: Database, + replica_id: u64, + guard: ®istry::WriteGuard, + ) -> anyhow::Result { let database_identity = database.database_identity; record_module_host_init_attempt(database_identity); - Host::try_init(self, database, replica_id) - .await + let result = Host::try_init(self, database, replica_id, guard.registration()).await; + if result.as_ref().is_err_and(lifecycle::writer_unconfirmed) { + guard.quarantine(); + } + result .inspect_err(|error| { let cause = HostInitError::metric_cause(error); match cause { @@ -1098,8 +1166,9 @@ fn repair_stale_view_backing_tables_on_launch(launched: &LaunchedModule) -> anyh /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise publish the complete environment with the module, including when -/// its program hash is unchanged. +/// Admission, including unchanged programs and idempotent deployment retries, +/// is serialized inside the migration transaction. Every accepted publication +/// carries a complete replacement environment, even for unchanged programs. async fn update_module( db: &RelationalDB, module: &ModuleHost, @@ -1107,20 +1176,29 @@ async fn update_module( old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { let addr = db.database_identity(); - let Some(stored) = stored_program_hash(db)? else { - bail!("database `{addr}` not yet initialized"); - }; - info!("publishing `{}` from {} to {}", addr, stored, program.hash); - // Even an unchanged program publishes a complete replacement environment. - module - .update_database_with_environment(program, old_module_info, policy, environment) - .await + match stored_program_hash(db)? { + None => Err(anyhow!("database `{addr}` not yet initialized")), + Some(stored) => { + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + module + .update_database_with_environment_and_deployment( + program, + old_module_info, + policy, + environment, + deployment, + ) + .await + } + } } /// Encapsulates a database, associated module, and auxiliary state. struct Host { + registration: Registration, /// The [`ModuleHost`], providing the callable reducer API. /// /// Modules may be updated via [`Host::update_module`]. @@ -1146,18 +1224,23 @@ struct Host { /// Handle to the task responsible for cleaning up old views. /// The task is aborted when [`Host`] is dropped. view_cleanup_task: AbortHandle, + + // Runtime that owns this host and its physical shutdown. + runtime: spacetimedb_runtime::Handle, } impl Host { /// Attempt to instantiate a [`Host`] from persistent storage. /// - /// Note that this does **not** run module initialization routines, but may - /// create on-disk artifacts if the host / database did not exist. + /// This executes the stored module and initializes a new database when + /// necessary. It may create on-disk artifacts. Retained cleanup uses the + /// separate metadata-only open path instead. #[tracing::instrument(level = "debug", skip_all)] async fn try_init( host_controller: &HostController, database: Database, replica_id: u64, + registration: Registration, ) -> anyhow::Result { let database_identity = database.database_identity; let HostController { @@ -1166,8 +1249,6 @@ impl Host { program_storage, energy_monitor, runtimes, - persistence, - page_pool, bsatn_rlb_pool, memory_observer, .. @@ -1177,245 +1258,239 @@ impl Host { let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(&runtime); let tx_metrics_recorder_task = guard(tx_metrics_recorder_task, |task| task.abort()); - let (db, connected_clients) = match config.storage { - db::Storage::Memory => RelationalDB::open( - database.database_identity, - database.owner_identity, - EmptyHistory::new(), - None, - Some(tx_metrics_queue), - page_pool.clone(), - )?, - db::Storage::Disk => { - // Replay from the local state. - let history = relational_db::local_history(&replica_dir, &runtime).await?; - let persistence_db = db::persistence::Database { - id: database.id, - database_identity: database.database_identity, - owner_identity: database.owner_identity, - }; - let persistence = persistence.persistence(&persistence_db, replica_id).await?; - // Loading a database from persistent storage involves heavy - // blocking I/O. `asyncify` to avoid blocking the async worker. - let (db, clients) = asyncify({ - let database_identity = database.database_identity; - let owner_identity = database.owner_identity; - let page_pool = page_pool.clone(); - move || { - RelationalDB::open( - database_identity, - owner_identity, - history, - Some(persistence), - Some(tx_metrics_queue), - page_pool, - ) - } - }) - .await - // Make sure we log the source chain of the error - // as a single line, with the help of `anyhow`. - .map_err(anyhow::Error::from) - .inspect_err(|e| { - tracing::error!( - database = %database.database_identity, - replica = replica_id, - "Failed to open database: {e:#}" - ); - })?; - - (db, clients) - } - }; - let db = db.with_memory_observer(memory_observer.clone()); - let (mut program, program_needs_init) = match db.program()? { - // Launch module with program from existing database. - Some(program) => { - info!( - "loaded program {} from the database host-type={}", - program.hash, - HostType::from(program.kind) - ); - (program, false) - } - // Database is empty, load program from external storage and run - // initialization. - None => { - info!( - "loading program {} from external storage host-type={}", - database.initial_program, database.host_type - ); - let program_bytes = load_program(program_storage, database.initial_program).await?; - let program = Program { - hash: database.initial_program, - bytes: program_bytes, - kind: database.host_type.into(), - }; - (program, true) - } - }; + let (db, connected_clients, joined) = + lifecycle::open_database(host_controller, &database, replica_id, Some(tx_metrics_queue), &runtime).await?; let bootstrap_generation = database.bootstrap_generation; - let initial_environment = if program_needs_init { - match &host_controller.initial_environment_source { - Some(source) => source.load(&database).await?, - None => Default::default(), - } - } else { - Default::default() - }; let mut bootstrap_completion = Some(BootstrapCompletion::durable(bootstrap_generation)); - - let relational_db = Arc::new(db); - let (program, launched) = match HostType::from(program.kind) { - HostType::Js => { - ModuleLauncher { - database, - replica_id, - program, - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db, - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + let initialized = std::panic::AssertUnwindSafe(async { + let (mut program, program_needs_init, initial_deployment) = match db.program()? { + // Launch module with program from existing database. + Some(program) => { + info!( + "loaded program {} from the database host-type={}", + program.hash, + HostType::from(program.kind) + ); + (program, false, None) } - .launch_module() - .await? - } - HostType::Wasm => { - // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 - // the host type in `st_module` was always set to wasm. - // We now correctly use the host type from the database, but the - // module may in fact be a JS module. - // So if launching it as a wasm module fails, try JS instead. - // If this succeeds, the module is definitely a JS module, so - // attempt to repair `st_module` in this case. - // - // TODO: This code should eventually be removed once all - // databases have been repaired. - let launch_wasm_result = ModuleLauncher { - database: database.clone(), - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.clone().module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + // Database is empty, load program from external storage and run + // initialization. + None => { + info!( + "loading program {} from external storage host-type={}", + database.initial_program, database.host_type + ); + let initial_deployment = program_storage.initial_deployment(&database).await?; + let program_bytes = load_program(program_storage, database.initial_program).await?; + let program = Program { + hash: database.initial_program, + bytes: program_bytes, + kind: database.host_type.into(), + }; + (program, true, initial_deployment) } - .launch_module() - .await; - match launch_wasm_result { - Ok(program_and_module_host) => program_and_module_host, - Err(e) => { - warn!("failed to launch wasm module, trying js: {e:#}"); - - program.kind = ModuleKind::JS; - let res = ModuleLauncher { - database, - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), - } - .launch_module() - .await; + }; - if res.is_ok() { - let _ = relational_db - .with_auto_commit(Workload::Internal, |tx| relational_db.update_program(tx, program)); - } + let initial_environment = if program_needs_init { + match &host_controller.initial_environment_source { + Some(source) => source.load(&database).await?, + None => Default::default(), + } + } else { + Default::default() + }; - res? + let relational_db = db.clone(); + let (program, launched) = match HostType::from(program.kind) { + HostType::Js => { + ModuleLauncher { + database, + replica_id, + program, + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db, + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), } + .launch_module() + .await? } - } - }; + HostType::Wasm => { + // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 + // the host type in `st_module` was always set to wasm. + // We now correctly use the host type from the database, but the + // module may in fact be a JS module. + // Retry JS only for an existing stored module whose database + // declaration explicitly identifies it as JS. A new publication + // or declared Wasm module must preserve its Wasm validation error. + // If the legacy retry succeeds, repair `st_module`. + // + // TODO: This code should eventually be removed once all + // databases have been repaired. + let launch_wasm_result = ModuleLauncher { + database: database.clone(), + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.clone().module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + match launch_wasm_result { + Ok(program_and_module_host) => program_and_module_host, + Err(e) => { + if program_needs_init || database.host_type != HostType::Js { + return Err(e); + } + warn!("failed to launch wasm module, trying js: {e:#}"); + + program.kind = ModuleKind::JS; + let res = ModuleLauncher { + database, + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + + if res.is_ok() { + let _ = relational_db.with_auto_commit(Workload::Internal, |tx| { + relational_db.update_program(tx, program) + }); + } + + res.map_err(|js_error| { + e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) + })? + } + } + } + }; - if program_needs_init { - let InitDatabaseResult { reducer, tx_offset } = launched - .module_host - .init_database_with_environment(program, initial_environment) - .await?; - if let Some(call_result) = reducer { - validate_init_reducer_call_result(call_result)?; + if program_needs_init { + let InitDatabaseResult { reducer, tx_offset } = launched + .module_host + .init_database_with_environment_and_deployment(program, initial_environment, initial_deployment) + .await?; + if let Some(call_result) = reducer { + validate_init_reducer_call_result(call_result)?; + } + bootstrap_completion = Some(BootstrapCompletion::pending( + bootstrap_generation, + tx_offset, + launched.module_host.durable_tx_offset(), + )); + } else { + repair_stale_view_backing_tables_on_launch(&launched)?; + drop(program) } - bootstrap_completion = Some(BootstrapCompletion::pending( - bootstrap_generation, - tx_offset, - launched.module_host.durable_tx_offset(), - )); - } else { - repair_stale_view_backing_tables_on_launch(&launched)?; - drop(program) - } - let LaunchedModule { - replica_ctx, - module_host, - scheduler, - scheduler_starter, - } = launched; - - // Disconnect dangling clients. - // No need to clear view tables here since we do it in `clear_all_clients`. - for (identity, connection_id) in connected_clients { - module_host - .call_identity_disconnected(identity, connection_id) - .await - .with_context(|| { - format!( - "Error calling disconnect for {} {} on {}", - identity, connection_id, replica_ctx.database_identity - ) - })?; - } - // We should have no clients left, but we do this just in case. - // This should only matter if we crashed with something in st_client_credentials, - // then restarted with an older version of the code that doesn't use st_client_credentials. - // That case would cause some permanently dangling st_client_credentials. - // Since we have no clients on startup, this should be safe to do regardless. - module_host.clear_all_clients().await?; - - scheduler_starter.start(&module_host)?; - let disk_metrics_recorder_task: spacetimedb_runtime::AbortHandle = - tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle().into(); - let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone(), &runtime); - - let module = watch::Sender::new(module_host); - let tx_metrics_recorder_task = scopeguard::ScopeGuard::into_inner(tx_metrics_recorder_task); - - Ok(HostInit { - host: Host { - module, + let LaunchedModule { replica_ctx, + module_host, scheduler, - disk_metrics_recorder_task, - tx_metrics_recorder_task, - view_cleanup_task, - }, - bootstrap_completion, + scheduler_starter, + } = launched; + + // Disconnect dangling clients. + // No need to clear view tables here since we do it in `clear_all_clients`. + for (identity, connection_id) in connected_clients { + module_host + .call_identity_disconnected(identity, connection_id) + .await + .with_context(|| { + format!( + "Error calling disconnect for {} {} on {}", + identity, connection_id, replica_ctx.database_identity + ) + })?; + } + // We should have no clients left, but we do this just in case. + // This should only matter if we crashed with something in st_client_credentials, + // then restarted with an older version of the code that doesn't use st_client_credentials. + // That case would cause some permanently dangling st_client_credentials. + // Since we have no clients on startup, this should be safe to do regardless. + module_host.clear_all_clients().await?; + + // Scheduled work can run before this Host is installed in its cell. + // Its callback must already identify this executable; the close + // owner will wait for our pinned guard before taking the Host. + registration.activate(); + scheduler_starter.start(&module_host)?; + #[cfg(test)] + if lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .remove(&replica_ctx.database_identity) + { + host_controller.unregister_fn(registration.clone(), database_identity)(); + } + let disk_metrics_recorder_task: AbortHandle = + tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle().into(); + let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone(), &runtime); + + let module = watch::Sender::new(module_host); + + let tx_metrics_recorder_task = scopeguard::ScopeGuard::into_inner(tx_metrics_recorder_task); + Ok(HostInit { + host: Host { + registration, + runtime: runtime.clone(), + module, + replica_ctx, + scheduler, + disk_metrics_recorder_task, + tx_metrics_recorder_task, + view_cleanup_task, + }, + bootstrap_completion, + }) }) + .catch_unwind() + .await; + match initialized { + Ok(Ok(host)) => Ok(host), + Ok(Err(error)) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + Err(error) + } + Err(panic) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + std::panic::resume_unwind(panic) + } + } } /// Construct an in-memory instance of `database` running `program`. @@ -1483,10 +1558,13 @@ impl Host { runtimes: Arc, program: Program, policy: MigrationPolicy, + environment: std::collections::BTreeMap, + deployment: Option, energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, + registration: Registration, core: AllocatedJobCore, - environment: std::collections::BTreeMap, + database_committed: &mut bool, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1512,53 +1590,76 @@ impl Host { old_module_info, policy, environment, + deployment, ) .await { Ok(result) => result, Err(error) => { - // This candidate was never installed or scheduled. Close its - // receiver first so cleanup cannot wait on an unstarted actor. + // An uninstalled candidate has no running scheduler. Release + // its receiver before positively closing the module actor. drop(scheduler_starter); module.exit().await; return Err(error); } }; + *database_committed = matches!( + update_result, + UpdateDatabaseResult::UpdatePerformed { .. } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + ); + + #[cfg(test)] + if *database_committed + && FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .remove(&replica_ctx.database_identity) + { + bail!("injected scheduler activation failure after deployment commit"); + } + // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. match update_result { UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed { .. } => { - self.scheduler = scheduler; + registration.activate(); scheduler_starter.start(&module)?; + self.scheduler = scheduler; + self.registration = registration.clone(); let old_module = self.module.send_replace(module); old_module.exit().await; } // In this case, we need to disconnect all clients connected to the old module UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } => { + self.registration = registration; + self.registration.activate(); // Replace the module first, so that new clients get the new module. let old_watcher = std::mem::replace(&mut self.module, watch::Sender::new(module.clone())); + let old_module = old_watcher.borrow().clone(); // Disconnect all clients connected to the old module. - let connected_clients = replica_ctx.relational_db().connected_clients()?; - for (identity, connection_id) in connected_clients { - let client_actor_id = ClientActorId { - identity, - connection_id, - name: ClientName(0), - }; - //NOTE: This will call disconnect reducer of the new module, not the old one. - //It makes sense, as relationaldb is already updated to the new module. - module.disconnect_client(client_actor_id).await; + let activation = async { + let connected_clients = replica_ctx.relational_db().connected_clients()?; + for (identity, connection_id) in connected_clients { + let client_actor_id = ClientActorId { + identity, + connection_id, + name: ClientName(0), + }; + // Disconnect uses the newly committed module. + module.disconnect_client(client_actor_id).await; + } + scheduler_starter.start(&module)?; + Ok::<_, anyhow::Error>(()) } - - self.scheduler = scheduler; - scheduler_starter.start(&module)?; + .await; // exit the old module, drop the `old_watcher` afterwards, // which will signal websocket clients that the module is gone. - let old_module = old_watcher.borrow().clone(); old_module.exit().await; + activation?; + self.scheduler = scheduler; } _ => { drop(scheduler_starter); @@ -1790,21 +1891,30 @@ mod tests { Arc::new(LocalPersistenceProvider::new(directory)), JobCores::without_pinned_cores(), ); - let cell = Arc::new(AsyncRwLock::new(None)); - controller.hosts.lock().insert(17, cell.clone()); - let accepted_reader = cell.clone().read_owned().await; - let mut close = tokio::spawn(async move { controller.exit_module_host_and_join(17).await }); + let accepted_reader = controller.acquire_read_lock(17).await.unwrap(); + let closing_controller = controller.clone(); + let mut close = tokio::spawn(async move { closing_controller.exit_module_host_and_join(17).await }); assert!(tokio::time::timeout(Duration::from_millis(30), &mut close) .await .is_err()); - // A deadline on the observer did not complete or discard the owned close. assert!(!close.is_finished()); + close.abort(); + assert!(close.await.unwrap_err().is_cancelled()); + // Neither a cancelled join waiter nor the legacy timed API can report + // positive closure while the accepted registry reader remains active. + assert!(controller.exit_module_host(17, Duration::from_millis(1)).await.is_err()); + let joining_controller = controller.clone(); + let mut joined = tokio::spawn(async move { joining_controller.exit_module_host_and_join(17).await }); + assert!(tokio::time::timeout(Duration::from_millis(30), &mut joined) + .await + .is_err()); drop(accepted_reader); - tokio::time::timeout(Duration::from_secs(5), close) + tokio::time::timeout(Duration::from_secs(5), joined) .await .unwrap() .unwrap() .unwrap(); + assert!(controller.hosts.lock().is_empty()); } fn reducer_call_result(outcome: ReducerOutcome) -> ReducerCallResult { @@ -1836,3 +1946,6 @@ mod tests { assert_eq!(HostInitError::metric_cause(&error), ModuleHostInitFailureCause::Other); } } + +#[cfg(test)] +mod invocation_flags_tests; diff --git a/crates/core/src/host/host_controller/deployment_tests.rs b/crates/core/src/host/host_controller/deployment_tests.rs new file mode 100644 index 00000000000..f853edf9056 --- /dev/null +++ b/crates/core/src/host/host_controller/deployment_tests.rs @@ -0,0 +1,489 @@ +use super::*; +use crate::db::deployment::{current_deployment, install_publication_fence}; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::empty_module; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ST_DEPLOYMENT_OPERATION_ID, ST_PUBLISH_FENCE_ID}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, Uuid}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct InitialStorage { + program: Program, + request: Option, + initial_lookups: AtomicUsize, +} + +#[async_trait] +impl ExternalStorage for InitialStorage { + async fn lookup(&self, hash: Hash) -> anyhow::Result>> { + Ok((self.program.hash == hash).then(|| self.program.bytes.clone())) + } + async fn initial_deployment(&self, _: &Database) -> anyhow::Result> { + self.initial_lookups.fetch_add(1, Ordering::SeqCst); + Ok(self.request.clone()) + } +} + +fn request(program: &Program, sequence: u64, initial: bool) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::ONE, + expected_revision: None, + expected_last_operation: None, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: if initial { + ModuleComponent::SystemEmpty(spacetimedb_lib::deployment::system_empty::empty().descriptor) + } else { + ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: program.hash, + }) + }, + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([4; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fixture(id: u64, bootstrap: bool) -> (tempfile::TempDir, HostController, Database, Arc) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(empty_module::VERSION).unwrap(); + let storage = Arc::new(InitialStorage { + request: bootstrap.then(|| request(&program, 1, true)), + program: program.clone(), + initial_lookups: AtomicUsize::new(0), + }); + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage.clone(), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database, storage) +} + +#[tokio::test(flavor = "multi_thread")] +async fn bootstrap_commits_fence_program_and_deployment_and_reopens_from_disk() { + let (_directory, controller, database, storage) = fixture(0xdd01, true); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let expected = storage.request.as_ref().unwrap().deployment.revision().unwrap(); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_PUBLISH_FENCE_ID), Some(1)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + assert_eq!( + module.relational_db().program().unwrap().unwrap().hash, + database.initial_program + ); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!( + storage.initial_lookups.load(Ordering::SeqCst), + 1, + "replay must not re-run bootstrap intent" + ); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn activation_failure_after_commit_closes_old_host_and_recovers_committed_program() { + let (_directory, controller, database, _storage) = fixture(0xdd02, false); + let old_module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let watcher = controller.watch_module_host(database.id).await.unwrap(); + let mut bytes = spacetimedb_lib::deployment::system_empty::empty().bytes.to_vec(); + bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer = Program::from_bytes(ModuleKind::WASM, bytes); + let publication = request(&newer, 1, false); + old_module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .insert(database.database_identity); + let error = controller + .update_module_host_with_deployment( + database.clone(), + HostType::Wasm, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + Some(publication.clone()), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("injected scheduler activation failure")); + assert!( + controller.get_module_host(database.id).await.is_err(), + "old executable must not remain available" + ); + assert!(watcher.has_changed().is_err(), "old client watcher must close"); + assert_eq!(old_module.relational_db().program().unwrap().unwrap().hash, newer.hash); + old_module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + }); + drop(watcher); + drop(old_module); + let recovered = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(recovered.info.module_hash, newer.hash); + recovered.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(recovered); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +struct DeclaredInitialEnvironment { + database: Database, + values: std::collections::BTreeMap, + loads: AtomicUsize, +} + +#[async_trait] +impl InitialEnvironmentSource for DeclaredInitialEnvironment { + async fn load(&self, database: &Database) -> anyhow::Result> { + anyhow::ensure!( + database.id == self.database.id + && database.database_identity == self.database.database_identity + && database.owner_identity == self.database.owner_identity + && database.initial_program == self.database.initial_program + && database.bootstrap_generation == self.database.bootstrap_generation, + "bootstrap configuration does not match the exact persisted database generation" + ); + self.loads.fetch_add(1, Ordering::SeqCst); + Ok(self.values.clone()) + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn declared_builtin_publishes_complete_environment_atomically_and_reopens() { + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use std::collections::BTreeMap; + + let schema = EnvironmentSchema::new(vec![ + EnvironmentDeclaration { + name: "REQUIRED".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }, + EnvironmentDeclaration { + name: "MODE".into(), + constraint: EnvironmentConstraint::OneOf(vec!["ready".into(), "paused".into()]), + optional: false, + }, + EnvironmentDeclaration { + name: "FIXED".into(), + constraint: EnvironmentConstraint::Literal("constant".into()), + optional: false, + }, + EnvironmentDeclaration { + name: "OPTIONAL".into(), + constraint: EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap(); + let initial_values = BTreeMap::from([ + ("REQUIRED".into(), "initial".into()), + ("MODE".into(), "ready".into()), + ("FIXED".into(), "constant".into()), + ("OPTIONAL".into(), "remove-on-next-publish".into()), + ]); + let program = empty_module::declared_program(&schema).unwrap(); + let database = Database { + id: 0xdd03, + database_identity: Identity::from_u256(0xdd03_u64.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + bootstrap_generation: 7, + }; + let descriptor = spacetimedb_lib::deployment::SystemEmptyModule { + version: empty_module::VERSION, + program_hash: program.hash, + }; + let mut initial = request(&program, 1, true); + let DeploymentSpec::V1(spec) = &mut initial.deployment; + spec.module = ModuleComponent::SystemEmpty(descriptor); + spec.container.as_mut().unwrap().env_keys = vec!["FIXED".into(), "MODE".into(), "REQUIRED".into()]; + let revision = initial.deployment.revision().unwrap(); + let storage = Arc::new(InitialStorage { + program: program.clone(), + request: Some(initial.clone()), + initial_lookups: AtomicUsize::new(0), + }); + let environment = Arc::new(DeclaredInitialEnvironment { + database: database.clone(), + values: initial_values.clone(), + loads: AtomicUsize::new(0), + }); + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage.clone(), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ) + .with_initial_environment_source(environment.clone()); + + let result = std::panic::AssertUnwindSafe(async { + let launched = controller + .get_or_launch_module_host_with_bootstrap(database.clone(), database.id) + .await + .unwrap(); + if let Some(completion) = launched.bootstrap_completion { + assert_eq!(completion.bootstrap_generation(), database.bootstrap_generation); + completion.wait().await.unwrap(); + } + let module = launched.module; + assert_eq!(module.info.module_def.environment(), &schema); + assert!(module.info.module_def.tables().next().is_none()); + let assert_state = |module: &ModuleHost, expected: &BTreeMap, receipts| { + assert!(empty_module::matches_program( + &descriptor, + &module.relational_db().program().unwrap().unwrap() + )); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(db::environment::snapshot(tx).unwrap(), *expected); + assert_eq!(current_deployment(tx).unwrap().unwrap().0, revision); + let cursor = db::deployment::current_publication(tx).unwrap().unwrap(); + assert_eq!(cursor.publication_epoch, receipts); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(receipts)); + }); + }; + assert_state(&module, &initial_values, 1); + let replacement = BTreeMap::from([ + ("REQUIRED".into(), "replaced".into()), + ("MODE".into(), "paused".into()), + ("FIXED".into(), "constant".into()), + ]); + let publish_request = |sequence, previous_operation| { + let mut publication = request(&program, sequence, true); + publication.deployment = initial.deployment.clone(); + publication.expected_revision = Some(revision); + publication.expected_last_operation = Some(previous_operation); + publication + }; + let publication = publish_request(2, initial.operation_id); + let accepted_operation = publication.operation_id; + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + let updated = controller + .update_module_host_with_environment_and_deployment( + database.clone(), + HostType::Wasm, + database.id, + program.bytes.clone(), + MigrationPolicy::Compatible, + replacement.clone(), + Some(publication), + ) + .await + .unwrap(); + let UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } = updated + else { + panic!("unchanged-code ENV publication must commit a replacement: {updated:?}"); + }; + let offset = tx_offset.await.unwrap(); + durable_offset.unwrap().wait_for(offset).await.unwrap(); + let module = controller.get_module_host(database.id).await.unwrap(); + assert_state(&module, &replacement, 2); + + // The same normalized deployment revision does not authorize replacing + // ENV prepared against the earlier committed operation. + let stale = publish_request(3, initial.operation_id); + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, stale.publication_epoch, stale.operation_id) + }) + .unwrap(); + assert!(controller + .update_module_host_with_environment_and_deployment( + database.clone(), + HostType::Wasm, + database.id, + program.bytes.clone(), + MigrationPolicy::Compatible, + initial_values.clone(), + Some(stale), + ) + .await + .is_err()); + assert_state(&controller.get_module_host(database.id).await.unwrap(), &replacement, 2); + + let mut invalid_sets = Vec::new(); + let mut invalid = replacement.clone(); + invalid.remove("REQUIRED"); + invalid_sets.push(invalid); + for (key, value) in [("MODE", "unknown"), ("FIXED", "changed"), ("UNDECLARED", "denied")] { + let mut invalid = replacement.clone(); + invalid.insert(key.into(), value.into()); + invalid_sets.push(invalid); + } + for (index, invalid) in invalid_sets.into_iter().enumerate() { + let publication = publish_request(4 + index as u64, accepted_operation); + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + assert!(controller + .update_module_host_with_environment_and_deployment( + database.clone(), + HostType::Wasm, + database.id, + program.bytes.clone(), + MigrationPolicy::Compatible, + invalid, + Some(publication), + ) + .await + .is_err()); + let current = controller.get_module_host(database.id).await.unwrap(); + assert_state(¤t, &replacement, 2); + } + // The exact builtin bytes remain part of admission even though a valid + // Wasm custom section would leave the extracted declarations unchanged. + let mut different_bytes = program.bytes.to_vec(); + different_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let publication = publish_request(8, accepted_operation); + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + assert!(controller + .update_module_host_with_environment_and_deployment( + database.clone(), + HostType::Wasm, + database.id, + different_bytes.into(), + MigrationPolicy::Compatible, + replacement.clone(), + Some(publication), + ) + .await + .is_err()); + assert_state(&controller.get_module_host(database.id).await.unwrap(), &replacement, 2); + drop(module); + controller.exit_module_host_and_join(database.id).await.unwrap(); + let reopened = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_state(&reopened, &replacement, 2); + assert_eq!(storage.initial_lookups.load(Ordering::SeqCst), 1); + assert_eq!(environment.loads.load(Ordering::SeqCst), 1); + }) + .catch_unwind() + .await; + // Always join the existing close owner before releasing its filesystem. + let cleanup = controller.exit_module_host_and_join(database.id).await; + if let Err(panic) = result { + if let Err(error) = cleanup { + log::error!("declared-builtin fixture cleanup failed after assertion: {error:#}"); + } + std::panic::resume_unwind(panic); + } + cleanup.unwrap(); +} diff --git a/crates/core/src/host/host_controller/execution_deadline_tests.rs b/crates/core/src/host/host_controller/execution_deadline_tests.rs new file mode 100644 index 00000000000..c1ca0ca1821 --- /dev/null +++ b/crates/core/src/host/host_controller/execution_deadline_tests.rs @@ -0,0 +1,413 @@ +//! Actual V8 module hosts, transactions and updates. No network or external service. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::FunctionArgs; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::db::raw_def::{v10::RawModuleDefV10Builder, v9::Lifecycle}; +use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_sats::{AlgebraicType, ProductType}; + +fn config() -> HostRuntimeConfig { + HostRuntimeConfig { + v8: V8Config { + execution_timeout: Duration::from_millis(200), + ..V8Config::default() + }, + ..HostRuntimeConfig::default() + } +} + +fn program(loop_in_first_init: bool) -> Program { + program_with_startup_cutoff(loop_in_first_init, u64::MAX) +} + +fn program_with_startup_cutoff(loop_in_first_init: bool, cutoff_millis: u64) -> Program { + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); + schema.add_reducer("loop", ProductType::unit()); + schema.add_reducer("good", ProductType::unit()); + schema.add_procedure("task", ProductType::unit(), AlgebraicType::U64); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{ register_hooks, table_id_from_name, datastore_insert_bsatn }} from "spacetime:sys@1.0"; + import {{ register_hooks as register_procedure_hooks }} from "spacetime:sys@1.2"; + if (Date.now() > {cutoff_millis}) {{ for (;;) {{}} }} + let initCalls = 0; + let nextValue = 0; + register_hooks({{ + __describe_module__: function() {{ return new Uint8Array({schema:?}); }}, + __call_reducer__: function(id) {{ + const row = new Uint8Array(8); + row[0] = ++nextValue; + datastore_insert_bsatn(table_id_from_name("rows"), row); + if (id === 1 || (id === 0 && ++initCalls === 1 && {loop_in_first_init})) {{ for (;;) {{}} }} + return {{ tag: "ok" }}; + }}, + }}); + let retained; + register_procedure_hooks({{ __call_procedure__: function() {{ + if (Date.now() > {cutoff_millis}) {{ retained = new Array(4 * 1024 * 1024).fill(1); }} + return new Uint8Array(8); + }} }}); + "# + ) + .into_bytes(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_procedure_startup_failure_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed05, + &program, + HostRuntimeConfig { + v8: V8Config { + // One slot makes a leaked admission permit observable on retry. + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "good").await.is_ok()); + let until_cutoff = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(until_cutoff + Duration::from_millis(50)).await; + // The module's main isolate already ran startup. Only newly created + // procedure isolates encounter the now-hostile startup branch. + for _ in 0..2 { + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + } + assert_eq!(row_count(&module), Some(4)); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +fn database(id: u64, program: &Program) -> Database { + Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Js, + initial_program: program.hash, + bootstrap_generation: 0, + } +} + +fn controller_fixture( + id: u64, + program: &Program, + config: HostRuntimeConfig, +) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let storage = move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }; + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + config, + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + (directory, controller, database(id, program)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_failed_procedure_recreation_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed06, + &program, + HostRuntimeConfig { + v8: V8Config { + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + heap_policy: crate::config::V8HeapPolicyConfig { + heap_limit_bytes: 64 * 1024 * 1024, + heap_check_request_interval: Some(1), + heap_gc_trigger_fraction: 0.2, + heap_retire_fraction: 0.2, + ..Default::default() + }, + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // Populate the one-slot procedure pool before the startup branch changes. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let remaining = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(remaining + Duration::from_millis(50)).await; + // This invocation runs in the existing isolate and leaves32MiB alive. The + // real post-call heap policy retires it and tries a new, now-looping startup. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!( + error.to_string().contains("procedure isolate startup failed"), + "{error}" + ); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + // A subsequent checkout also completes, proving the dead instance's slot + // was released instead of leaked after its replacement failed. + assert!(timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary,) + ) + .await + .unwrap() + .result + .is_err()); + assert!(call(®istered, "good").await.is_ok()); + drop(registered); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +async fn launch(id: u64, program: Program) -> anyhow::Result<(Program, LaunchedModule)> { + timeout( + Duration::from_secs(5), + Host::try_init_in_memory_to_check( + &HostRuntimes::new(None, config()), + PagePool::new(None), + database(id, &program), + program, + AllocatedJobCore::default(), + BsatnRowListBuilderPool::new(), + ), + ) + .await? +} + +fn row_count(module: &ModuleHost) -> Option { + module.relational_db().with_read_only(Workload::Internal, |tx| { + tx.table_id_from_name("rows") + .unwrap() + .and_then(|table| tx.table_row_count(table)) + }) +} + +async fn call(module: &ModuleHost, name: &str) -> ReducerCallResult { + timeout( + Duration::from_secs(5), + module.call_reducer(Identity::ONE, None, None, None, None, name, FunctionArgs::Nullary), + ) + .await + .unwrap() + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_rolls_back_init_and_reducer_and_preserves_isolate() { + let (program, launched) = launch(0xed01, program(true)).await.unwrap(); + let module = launched.module_host; + let result = module.init_database(program.clone()).await.unwrap().reducer.unwrap(); + assert!(result.is_err(), "infinite init must fail"); + assert!(module.relational_db().program().unwrap().is_none()); + assert_eq!( + row_count(&module), + None, + "failed init must roll back the schema and inserted row" + ); + + assert!(module.init_database(program).await.unwrap().reducer.unwrap().is_ok()); + assert_eq!(row_count(&module), Some(1)); + let failed = call(&module, "loop").await; + assert!(failed.is_err()); + assert_eq!( + row_count(&module), + Some(1), + "timed-out reducer must roll back its write" + ); + + for _ in 0..64 { + assert!(call(&module, "good").await.is_ok()); + } + // Cross the original timer boundary before reusing the same isolate again. + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(call(&module, "good").await.is_ok()); + assert_eq!(row_count(&module), Some(66)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_bounds_startup_description_and_failed_update() { + let valid = program(false); + let looping_startup = Program::from_bytes(ModuleKind::JS, b"for (;;) {}".to_vec()); + let looping_description = Program::from_bytes( + ModuleKind::JS, + br#"import {register_hooks} from "spacetime:sys@1.0"; + register_hooks({__describe_module__: function() {for (;;) {}}, __call_reducer__: function() {}});"# + .to_vec(), + ); + for (id, bad) in [(0xed02, &looping_startup), (0xed03, &looping_description)] { + let error = match launch(id, bad.clone()).await { + Ok(_) => panic!("infinite JavaScript unexpectedly launched"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("wall-clock limit"), "{error:#}"); + } + + let (_directory, controller, database) = controller_fixture(0xed04, &valid, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(row_count(&module), Some(1)); + for bad in [looping_startup, looping_description] { + assert!(timeout( + Duration::from_secs(5), + controller.update_module_host( + database.clone(), + HostType::Js, + database.id, + bad.bytes, + MigrationPolicy::Compatible, + ) + ) + .await + .unwrap() + .is_err()); + assert_eq!(module.relational_db().program().unwrap().unwrap().hash, valid.hash); + assert!(call(&module, "good").await.is_ok()); + } + let mut newer = valid.bytes.to_vec(); + newer.extend_from_slice(b"\n// Valid replacement after two deadline failures.\n"); + let newer = Program::from_bytes(ModuleKind::JS, newer); + controller + .update_module_host( + database.clone(), + HostType::Js, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + let replacement = controller.get_module_host(database.id).await.unwrap(); + assert_eq!(replacement.relational_db().program().unwrap().unwrap().hash, newer.hash); + assert!(call(&replacement, "good").await.is_ok()); + drop(replacement); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn javascript_logging_handles_direct_startup_and_preserves_wrapped_call_locations() { + use futures::TryStreamExt as _; + + let mut schema = RawModuleDefV10Builder::new(); + schema.add_reducer("log", ProductType::unit()); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + // Direct top-level logging has exactly one JS frame. It previously passed + // index1 to V8's unchecked GetFrame and could crash the entire host process. + let source = format!( + r#"import {{ register_hooks, console_log }} from "spacetime:sys@1.0"; +console_log(2, "direct-startup"); +function wrapped(message) {{ console_log(2, message); }} +wrapped("wrapped-startup"); +register_hooks({{ + __describe_module__: () => new Uint8Array({schema:?}), + __call_reducer__: () => {{ + console_log(2, "direct-reducer"); + wrapped("wrapped-reducer"); + return {{ tag: "ok" }}; + }}, +}}); +"# + ); + let program = Program::from_bytes(ModuleKind::JS, source.into_bytes()); + let (_directory, controller, database) = controller_fixture(0xed08, &program, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "log").await.is_ok()); + let chunks: Vec<_> = module + .database_logger() + .tail(None, false) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let bytes = chunks.into_iter().flatten().collect::>(); + let records = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + for (message, line) in [ + ("direct-startup", 2), + ("wrapped-startup", 4), + ("direct-reducer", 8), + ("wrapped-reducer", 9), + ] { + let record = records.iter().find(|record| record["message"] == message).unwrap(); + assert_eq!(record["line_number"], line, "wrong call location for {message}"); + assert!(record["filename"].as_str().is_some_and(|filename| !filename.is_empty())); + } +} diff --git a/crates/core/src/host/host_controller/lifecycle.rs b/crates/core/src/host/host_controller/lifecycle.rs new file mode 100644 index 00000000000..e29871a66d2 --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle.rs @@ -0,0 +1,157 @@ +//! Shared physical writer completion for ordinary module initialization and +//! shutdown. Provider snapshot and archival services keep their own ownership. + +use super::*; +use crate::db::persistence::Persistence; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use std::panic::{resume_unwind, AssertUnwindSafe}; +use std::sync::OnceLock; + +#[derive(Debug, thiserror::Error)] +pub(super) enum CloseFailure { + #[error("module exit panicked; the storage writer was positively closed")] + ModuleExit, + #[error("storage writer close panicked; the replica remains quarantined")] + WriterUnconfirmed, +} + +#[cfg(test)] +pub(super) static FAIL_NEXT_MODULE_EXIT: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +#[cfg(test)] +pub(super) static PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +pub(super) fn writer_unconfirmed(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(CloseFailure::WriterUnconfirmed) + ) +} + +/// Some providers hand the first close caller the writer's JoinHandle and let +/// later calls finish immediately. Share that *first* physical close instead. +/// In particular, partial RelationalDB replay may start close from Drop before +/// our owning task sees its error. That task must join the same close future. +pub(super) struct JoinedDurability { + inner: Arc>, + close: OnceLock>>>, +} + +impl JoinedDurability { + pub fn wrap(persistence: &mut Persistence) -> Arc { + let joined = Arc::new(Self { + inner: persistence.durability.clone(), + close: OnceLock::new(), + }); + persistence.durability = joined.clone(); + joined + } + + pub async fn join(&self) -> Result, CloseFailure> { + AssertUnwindSafe(async { self.close().await }) + .catch_unwind() + .await + .map_err(|_| CloseFailure::WriterUnconfirmed) + } +} + +impl Durability for JoinedDurability { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + self.close.get_or_init(|| self.inner.close().shared()).clone().boxed() + } +} + +/// A failed replay joins the same physical writer close before another normal +/// initialization can acquire the replica's canonical registry cell. +pub(super) async fn open_database( + controller: &HostController, + database: &Database, + replica_id: u64, + tx_metrics_queue: Option, + runtime: &spacetimedb_runtime::Handle, +) -> anyhow::Result<( + Arc, + relational_db::ConnectedClients, + Option>, +)> { + if matches!(controller.default_config.storage, db::Storage::Memory) { + let (db, clients) = RelationalDB::open( + database.database_identity, + database.owner_identity, + EmptyHistory::new(), + None, + tx_metrics_queue, + controller.page_pool.clone(), + )?; + return Ok(( + Arc::new(db.with_memory_observer(controller.memory_observer.clone())), + clients, + None, + )); + } + + let replica_dir = controller.data_dir.replica(replica_id); + let history = relational_db::local_history(&replica_dir, runtime).await?; + let persistence_db = crate::db::persistence::Database { + id: database.id, + database_identity: database.database_identity, + owner_identity: database.owner_identity, + }; + let mut persistence = controller.persistence.persistence(&persistence_db, replica_id).await?; + let joined = JoinedDurability::wrap(&mut persistence); + let identity = database.database_identity; + let owner = database.owner_identity; + let page_pool = controller.page_pool.clone(); + let opened = AssertUnwindSafe(asyncify(move || { + RelationalDB::open(identity, owner, history, Some(persistence), tx_metrics_queue, page_pool) + })) + .catch_unwind() + .await; + let (db, clients) = match opened { + Ok(Ok(opened)) => opened, + Ok(Err(error)) => { + joined.join().await?; + return Err(error.into()); + } + Err(panic) => { + joined.join().await?; + resume_unwind(panic); + } + }; + let db = Arc::new(db.with_memory_observer(controller.memory_observer.clone())); + Ok((db, clients, Some(joined))) +} + +pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { + let module = host.module.borrow().clone(); + let info = module.info(); + let identity = info.database_identity; + let table_names = info.module_def.tables().map(|table| table.name.deref()); + defer!(remove_database_gauges(&identity, table_names)); + let exited = AssertUnwindSafe(async { + module.exit().await; + #[cfg(test)] + if FAIL_NEXT_MODULE_EXIT.lock().remove(&identity) { + panic!("injected module exit failure"); + } + }) + .catch_unwind() + .await; + let writer = AssertUnwindSafe(module.relational_db().shutdown_with_runtime(&host.runtime)) + .catch_unwind() + .await; + drop(host); + writer.map_err(|_| CloseFailure::WriterUnconfirmed)?; + exited.map_err(|_| CloseFailure::ModuleExit) +} diff --git a/crates/core/src/host/host_controller/lifecycle_tests.rs b/crates/core/src/host/host_controller/lifecycle_tests.rs new file mode 100644 index 00000000000..3638365037b --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle_tests.rs @@ -0,0 +1,403 @@ +//! Real local durability writers and module hosts. No network or external +//! configuration. The probe counts writers until their first close completes. +use super::*; +use crate::db::persistence::{LocalPersistenceProvider, Persistence}; +use crate::host::empty_module; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Semaphore; + +struct Probe { + opened: AtomicUsize, + active: AtomicUsize, + maximum: AtomicUsize, + block_close: AtomicBool, + panic_close: AtomicBool, + close_started: Semaphore, + release_close: Semaphore, +} + +impl Default for Probe { + fn default() -> Self { + Self { + opened: AtomicUsize::new(0), + active: AtomicUsize::new(0), + maximum: AtomicUsize::new(0), + block_close: AtomicBool::new(false), + panic_close: AtomicBool::new(false), + close_started: Semaphore::new(0), + release_close: Semaphore::new(0), + } + } +} + +struct ProbedProvider { + local: LocalPersistenceProvider, + probe: Arc, +} + +#[async_trait] +impl PersistenceProvider for ProbedProvider { + async fn persistence( + &self, + database: &crate::db::persistence::Database, + replica: u64, + ) -> anyhow::Result { + let mut persistence = self.local.persistence(database, replica).await?; + let active = self.probe.active.fetch_add(1, Ordering::SeqCst) + 1; + self.probe.maximum.fetch_max(active, Ordering::SeqCst); + self.probe.opened.fetch_add(1, Ordering::SeqCst); + persistence.durability = Arc::new(ProbedWriter { + inner: persistence.durability, + probe: self.probe.clone(), + closing: AtomicBool::new(false), + }); + Ok(persistence) + } +} + +struct ProbedWriter { + inner: Arc>, + probe: Arc, + closing: AtomicBool, +} + +impl Durability for ProbedWriter { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + // Reproduce the actual first-caller-owns-join behavior deliberately. + // A second close must not let another database open before this one. + if self.closing.swap(true, Ordering::SeqCst) { + let offset = self.inner.durable_tx_offset().last_seen(); + return async move { offset }.boxed(); + } + let close = self.inner.close(); + let probe = self.probe.clone(); + async move { + probe.close_started.add_permits(1); + if probe.block_close.load(Ordering::SeqCst) { + probe.release_close.acquire().await.unwrap().forget(); + } + assert!( + !probe.panic_close.load(Ordering::SeqCst), + "injected writer close failure" + ); + let offset = close.await; + probe.active.fetch_sub(1, Ordering::SeqCst); + offset + } + .boxed() + } +} + +fn fixture( + id: u64, +) -> ( + tempfile::TempDir, + HostController, + Database, + Arc, + Arc, +) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(empty_module::VERSION).unwrap(); + let initial = program.clone(); + let lookups = Arc::new(AtomicUsize::new(0)); + let lookup_count = lookups.clone(); + let storage = move |hash| { + let initial = initial.clone(); + lookup_count.fetch_add(1, Ordering::SeqCst); + async move { Ok((hash == initial.hash).then_some(initial.bytes)) } + }; + let probe = Arc::new(Probe::default()); + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(ProbedProvider { + local: LocalPersistenceProvider::new(data), + probe: probe.clone(), + }), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database, probe, lookups) +} + +async fn closed_seed(controller: &HostController, database: &Database, probe: &Probe) { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + probe.close_started.acquire().await.unwrap().forget(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +async fn wait_for_close(probe: &Probe) { + timeout(Duration::from_secs(5), probe.close_started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_idle_module_and_queued_relookup_do_not_break_positive_close() { + let (_directory, controller, database, probe, _) = fixture(0xc002); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.block_close.store(true, Ordering::SeqCst); + let exiting = { + let controller = controller.clone(); + tokio::spawn(async move { controller.exit_module_host(0xc002, Duration::from_secs(5)).await }) + }; + wait_for_close(&probe).await; + let lookup = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!( + controller + .exit_module_host(database.id, Duration::from_millis(10)) + .await + .is_err(), + "a timeout is not positive closure" + ); + exiting.abort(); + assert!(exiting.await.unwrap_err().is_cancelled()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + probe.release_close.add_permits(1); + let current = timeout(Duration::from_secs(5), lookup).await.unwrap().unwrap().unwrap(); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert!(!Arc::ptr_eq(idle.relational_db(), current.relational_db())); + let seen_live = controller.get_module_host(database.id).await.unwrap(); + assert!(Arc::ptr_eq(seen_live.relational_db(), current.relational_db())); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + drop((idle, current)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_failed_normal_initialization_joins_writer_before_retry() { + let (_directory, controller, database, probe, _) = fixture(0xc008); + closed_seed(&controller, &database, &probe).await; + let mut wrong_owner = database.clone(); + wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); + probe.block_close.store(true, Ordering::SeqCst); + let failed = { + let controller = controller.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(wrong_owner.clone(), wrong_owner.id) + .await + }) + }; + wait_for_close(&probe).await; + let next = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!(!failed.is_finished()); + probe.release_close.add_permits(1); + assert!(failed.await.unwrap().is_err()); + let reopened = next.await.unwrap().unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + drop(reopened); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { + let (_directory, controller, database, probe, _) = fixture(0xc00a); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + lifecycle::FAIL_NEXT_MODULE_EXIT + .lock() + .insert(database.database_identity); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("module exit panicked")); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_unconfirmed_writer_close_reports_error_and_quarantines_replica() { + let (_directory, controller, database, probe, _) = fixture(0xc00b); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.panic_close.store(true, Ordering::SeqCst); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("writer close panicked")); + let error = timeout( + Duration::from_secs(1), + controller.get_or_launch_module_host(database.clone(), database.id), + ) + .await + .unwrap() + .expect_err("quarantined host must not reopen"); + assert!(error.to_string().contains("unable to lock")); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert!(controller + .exit_module_host(database.id, Duration::from_secs(1)) + .await + .unwrap_err() + .to_string() + .contains("quarantined")); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(module); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_panic_callback_before_initial_host_install_still_owns_cleanup() { + let (_directory, controller, database, probe, _) = fixture(0xc00c); + lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .insert(database.database_identity); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // No explicit exit is requested until the injected scheduler interleaving + // has independently started the actual storage writer close. + wait_for_close(&probe).await; + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { + let (_directory, controller, database, probe, _) = fixture(0xc006); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let old_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone(), database.database_identity) + }; + let mut newer = spacetimedb_lib::deployment::system_empty::empty().bytes.to_vec(); + newer.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer_hash = spacetimedb_lib::hash_bytes(&newer); + controller + .update_module_host( + database.clone(), + HostType::Wasm, + database.id, + newer.into(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + old_callback(); + assert_eq!( + controller.get_module_host(database.id).await.unwrap().info.module_hash, + newer_hash + ); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + let current_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone(), database.database_identity) + }; + current_callback(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + current_callback(); + assert!(controller.get_module_host(database.id).await.is_ok()); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); +} diff --git a/crates/core/src/host/host_controller/registry.rs b/crates/core/src/host/host_controller/registry.rs new file mode 100644 index 00000000000..030a1d7c3fe --- /dev/null +++ b/crates/core/src/host/host_controller/registry.rs @@ -0,0 +1,333 @@ +//! Canonical host cells, pinned by queued and active operations. +//! +//! Lock order: the registry mutex is never held across an await. A cell guard +//! may briefly acquire the registry mutex to publish its resident-host state. +//! It releases the cell guard before releasing its operation pin. The last pin +//! removes an empty, non-closing entry without waiting for unrelated Arc owners. + +use super::{Host, HostCell}; +use parking_lot::Mutex; +use spacetimedb_data_structures::map::IntMap; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, Weak}; +use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard}; + +pub(super) type Hosts = Arc>>; + +pub(super) struct Entry { + cell: HostCell, + pins: usize, + resident: bool, + registration: Option>, + closing: Option>, +} + +/// Completion means the storage writer has actually closed, not that the +/// caller's wait expired. Provider-owned snapshot/archival services are separate. +pub(super) struct Closing { + result: watch::Sender>>>, +} + +impl Closing { + pub async fn wait(&self) -> anyhow::Result<()> { + let mut receiver = self.result.subscribe(); + let result = receiver.wait_for(Option::is_some).await?.clone().unwrap(); + result.map_err(|message| anyhow::anyhow!(message.to_string())) + } +} + +/// A pin counts an operation, including time queued for the cell lock. Idle +/// ModuleHost references are not pins. Raw cells never authorize an operation. +pub(super) struct Pin { + hosts: Weak>>, + replica: u64, + cell: HostCell, +} + +impl Pin { + pub fn acquire(hosts: &Hosts, replica: u64) -> Self { + let mut entries = hosts.lock(); + let entry = entries.entry(replica).or_insert_with(|| Entry { + cell: HostCell::default(), + pins: 0, + resident: false, + registration: None, + closing: None, + }); + entry.pins += 1; + Self { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + } + } + + fn closing(&self) -> Option> { + let hosts = self.hosts.upgrade()?; + let entries = hosts.lock(); + let entry = entries.get(&self.replica)?; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.closing.clone() + } + + pub async fn read(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().read_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(ReadGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + pub async fn write(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().write_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(WriteGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + fn publish(&self, host: Option<&Host>) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.resident = host.is_some(); + entry.registration = host.map(|host| host.registration.token.clone()); + } + + fn registration(&self) -> Registration { + Registration { + hosts: self.hosts.clone(), + replica: self.replica, + cell: Arc::downgrade(&self.cell), + token: Arc::new(()), + } + } +} + +impl Drop for Pin { + fn drop(&mut self) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let removed = { + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + assert!(Arc::ptr_eq(&entry.cell, &self.cell), "replaced a pinned host cell"); + entry.pins -= 1; + if entry.pins == 0 && !entry.resident && entry.closing.is_none() { + entries.remove(&self.replica) + } else { + None + } + }; + drop(removed); + } +} + +pub(super) struct ReadGuard { + guard: Option>>, + pin: Option, +} + +impl Deref for ReadGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} + +impl Drop for ReadGuard { + fn drop(&mut self) { + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +pub(super) struct WriteGuard { + guard: Option>>, + pin: Option, +} + +impl WriteGuard { + pub fn registration(&self) -> Registration { + self.pin.as_ref().unwrap().registration() + } + + pub fn install(&mut self, host: Host) { + **self = Some(host); + self.pin.as_ref().unwrap().publish(self.as_ref()); + } + + pub fn quarantine(&self) { + let pin = self.pin.as_ref().unwrap(); + let Some(hosts) = pin.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&pin.replica).expect("pinned cell exists"); + if let Some(closing) = &entry.closing { + closing + .result + .send_replace(Some(Err("storage writer close is unconfirmed".into()))); + } + entry.closing = Some(Arc::new(Closing { + result: watch::Sender::new(Some(Err( + "storage writer close is unconfirmed; replica is quarantined".into() + ))), + })); + } +} + +impl Deref for WriteGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} +impl DerefMut for WriteGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.as_mut().unwrap() + } +} +impl Drop for WriteGuard { + fn drop(&mut self) { + self.pin.as_ref().unwrap().publish(self.as_ref()); + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +/// The token changes with each installed executable, even when its cell and +/// database stay the same. A late panic from an old executable is not authority +/// to close its replacement. +#[derive(Clone)] +pub(super) struct Registration { + hosts: Weak>>, + replica: u64, + cell: Weak>>, + token: Arc<()>, +} + +impl Registration { + pub fn activate(&self) { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return; + }; + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.replica) + && Arc::ptr_eq(&entry.cell, &cell) + { + entry.registration = Some(self.token.clone()); + } + } + + pub fn close_if_current(&self) -> Option { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return None; + }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&self.replica)?; + if !Arc::ptr_eq(&entry.cell, &cell) + || !entry + .registration + .as_ref() + .is_some_and(|token| Arc::ptr_eq(token, &self.token)) + { + return None; + } + Some(request_close(&hosts, self.replica, entry)) + } +} + +pub(super) struct CloseOwner { + pin: Pin, + completion: Arc, +} + +pub(super) struct CloseRequest { + pub completion: Arc, + pub owner: Option, +} + +pub(super) fn close(hosts: &Hosts, replica: u64) -> Option { + let mut entries = hosts.lock(); + let entry = entries.get_mut(&replica)?; + Some(request_close(hosts, replica, entry)) +} + +fn request_close(hosts: &Hosts, replica: u64, entry: &mut Entry) -> CloseRequest { + if let Some(completion) = &entry.closing { + return CloseRequest { + completion: completion.clone(), + owner: None, + }; + } + let completion = Arc::new(Closing { + result: watch::Sender::new(None), + }); + entry.closing = Some(completion.clone()); + entry.pins += 1; + CloseRequest { + completion: completion.clone(), + owner: Some(CloseOwner { + pin: Pin { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + }, + completion, + }), + } +} + +impl CloseOwner { + pub async fn run(self) { + let mut guard = self.pin.cell.clone().write_owned().await; + // A prior cold owner can report an unconfirmed writer before this + // queued close obtains its guard. An empty cell is not proof of closure. + if self.completion.result.borrow().is_some() { + return; + } + let result = match guard.take() { + Some(host) => super::lifecycle::close_host(host).await, + None => Ok(()), + }; + let writer_closed = !matches!(result, Err(super::lifecycle::CloseFailure::WriterUnconfirmed)); + // Publish under the cell lock, then release it before final registry + // unpin/removal. Closing stays set during both steps, so reopen waits. + self.pin.publish(None); + drop(guard); + if let Some(hosts) = self.pin.hosts.upgrade() { + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.pin.replica) { + debug_assert!(Arc::ptr_eq(&entry.cell, &self.pin.cell)); + if writer_closed { + entry.closing = None; + } + } + } + self.completion + .result + .send_replace(Some(result.map_err(|error| Arc::from(error.to_string())))); + // self.pin drops last. An unrelated idle Arc cannot delay completion. + } +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 00fce9c714f..ab1c411f3ec 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,4 +1,6 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; @@ -50,6 +52,10 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Set by trusted host dispatch for this invocation, independently of JWTs + /// and connection IDs. Cleared before each new function call. + call_auth_flags: u32, + hosted_auth: Option>, /// Bound by the host after validating this instance's module metadata. environment_module: Option<(spacetimedb_lib::Hash, Arc)>, environment_call_active: bool, @@ -57,7 +63,6 @@ pub struct InstanceEnv { in_anon_tx: bool, /// A procedure's last known transaction offset. procedure_last_tx_offset: Option, - call_auth_flags: u32, } /// `InstanceEnv` needs to be `Send` because it is created on the host thread @@ -240,11 +245,12 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + call_auth_flags: 0, + hosted_auth: None, environment_module: None, environment_call_active: false, in_anon_tx: false, procedure_last_tx_offset: None, - call_auth_flags: 0, } } @@ -261,12 +267,17 @@ impl InstanceEnv { self.func_name = Some(name); self.environment_call_active = true; self.call_auth_flags = 0; + self.hosted_auth = None; } pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { self.call_auth_flags = flags; } + pub(crate) fn set_hosted_auth(&mut self, auth: Option>) { + self.hosted_auth = auth; + } + pub(crate) fn get_call_auth_flags(&self) -> u32 { self.call_auth_flags } @@ -337,8 +348,11 @@ impl InstanceEnv { if !matches!(self.func_type, FuncCallType::Procedure) { return Err(NodesError::NotInTransaction); } - self.relational_db() - .with_read_only(Workload::Internal, |tx| self.read_declared_environment(tx, key)) + self.relational_db().with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, self.relational_db(), self.hosted_auth.as_deref()) + .map_err(|err| NodesError::HostedInvocationRejected(err.to_string()))?; + self.read_declared_environment(tx, key) + }) } fn read_declared_environment(&self, state: &impl StateView, key: &str) -> Result, NodesError> { @@ -374,8 +388,15 @@ impl InstanceEnv { } pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { - let tx = &mut *self.get_tx()?; - Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) + if let Ok(tx) = self.get_tx() { + return Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?); + } + // Procedures may inspect authentication before opening their first + // transaction. Use a short read transaction without manufacturing an + // internal caller or dropping the real connection's JWT. + Ok(self.relational_db().with_read_only(Workload::Internal, |tx| { + tx.get_jwt_payload(connection_id).map_err(DBError::from) + })?) } #[tracing::instrument(level = "trace", skip_all)] @@ -447,7 +468,8 @@ impl InstanceEnv { count } - /// Environment values are reachable only through their dedicated host interface. + /// Engine-owned deployment/authentication records and environment values + /// are reachable only through their dedicated host interfaces. fn require_module_table(table_id: TableId) -> Result<(), NodesError> { if is_module_restricted_table(table_id) { Err(NodesError::TableNotFound) @@ -853,6 +875,10 @@ impl InstanceEnv { let tx = self .relational_db() .begin_mut_tx(IsolationLevel::Serializable, Workload::Procedure); + if let Err(err) = check_hosted_admission(&tx, self.relational_db(), self.hosted_auth.as_deref()) { + let _ = tx.rollback(); + return Err(NodesError::HostedInvocationRejected(err.to_string())); + } self.tx.set_raw(tx); self.in_anon_tx = true; @@ -1702,12 +1728,113 @@ mod test { } #[test] - fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { - use spacetimedb_datastore::system_tables::ST_ENV_ID; + fn procedure_environment_snapshot_rechecks_durable_fence_and_expiry() -> Result<()> { + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + use crate::db::{deployment::install_container_fence, environment}; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + use std::time::SystemTime; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + let program = bind_test_environment(&mut env)?; + let schema = env.environment_module.as_ref().unwrap().1.environment().clone(); + env.start_funcall( + NamespacedIdentifier::from(spacetimedb_schema::identifier::Identifier::new("procedure".into())?), + Timestamp::now(), + FuncCallType::Procedure, + ); + let keys = JwtKeys::generate()?; + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public)])?; + let mint = |issued: SystemTime| -> Result<_> { + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: issued + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "platform.test", + &binding, + issued, + issued + Duration::from_secs(20), + "env-test", + )?; + Ok(Arc::new(validator.validate_token( + &token, + db.database_identity(), + issued, + |_, _, _| Some(binding), + )?)) + }; + let fence = |generation, allowed| StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: generation, + target_set_hash: Hash::ZERO, + allowed, + }; + db.with_auto_commit(Workload::ForTests, |tx| -> Result<()> { + install_container_fence(&db, tx, &fence(1, true))?; + db.update_program(tx, program.clone())?; + environment::replace( + &db, + tx, + &schema, + &std::collections::BTreeMap::from([("A".into(), "available".into())]), + )?; + Ok(()) + })?; + db.with_read_only(Workload::ForTests, |_| db.hosted_admission().begin()?.complete())?; + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + assert_eq!(env.env_get("A")?.as_deref(), Some("available")); + env.set_hosted_auth(Some(mint(SystemTime::now() - Duration::from_secs(60))?)); + assert!(matches!(env.env_get("A"), Err(NodesError::HostedInvocationRejected(_)))); + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(2, false)) + })?; + // Reconciled database admission does not override a denied source fence. + db.with_read_only(Workload::ForTests, |_| db.hosted_admission().begin()?.complete())?; + assert!(matches!(env.env_get("A"), Err(NodesError::HostedInvocationRejected(_)))); + env.set_hosted_auth(None); + assert_eq!(env.env_get("A")?.as_deref(), Some("available")); + Ok(()) + } + + /// Generate a `ProductValue` for use in [create_table_with_index] + fn product_row(i: usize) -> ProductValue { + let str = i.to_string(); + let str = str.repeat(i); + let id = i as u64; + product!(id, str) + } + + #[test] + fn module_cannot_access_hosted_system_records_by_guessed_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, + }; let db = relational_db()?; let (env, _runtime) = instance_env(db.clone())?; let mut slot = env.tx.clone(); - let protected = [(ST_ENV_ID, "st_env", to_vec("TOKEN")?)]; + let protected = [ + (ST_ENV_ID, "st_env", to_vec("TOKEN")?), + (ST_DEPLOYMENT_ID, "st_deployment", to_vec(&0u8)?), + (ST_PUBLISH_FENCE_ID, "st_publish_fence", to_vec(&0u8)?), + (ST_DEPLOYMENT_OPERATION_ID, "st_deployment_operation", to_vec(&0u128)?), + (ST_CONNECTION_AUTH_ID, "st_connection_auth", to_vec(&0u128)?), + (ST_CONTAINER_ENVIRONMENT_ID, "st_container_environment", to_vec(&0u64)?), + ( + ST_CONTAINER_FENCE_ID, + "st_container_fence", + to_vec(&spacetimedb_sats::u256::ZERO)?, + ), + ]; let tx = begin_mut_tx(&db); let (tx, result) = slot.set(tx, || -> Result<()> { for (table, name, point) in &protected { @@ -1772,14 +1899,6 @@ mod test { result } - /// Generate a `ProductValue` for use in [create_table_with_index] - fn product_row(i: usize) -> ProductValue { - let str = i.to_string(); - let str = str.repeat(i); - let id = i as u64; - product!(id, str) - } - /// Generate a BSATN encoded row for use in [create_table_with_index] fn bsatn_row(i: usize) -> Result> { Ok(to_vec(&product_row(i))?) diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 66677161fe1..14c876b94a3 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -10,7 +10,10 @@ use spacetimedb_lib::ProductValue; use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; +pub mod container_environment; +pub mod container_fence; mod disk_storage; +pub mod empty_module; mod host_controller; mod module_common; #[allow(clippy::too_many_arguments)] @@ -191,8 +194,8 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, - EnvGet, GetCallAuthFlags, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index 49f1126b54b..5bd4e61d927 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -19,7 +19,7 @@ use std::sync::Arc; pub fn build_common_module_from_raw( mcc: ModuleCreationContext, raw_def: RawModuleDef, -) -> Result { +) -> Result> { // Perform a bunch of validation on the raw definition. let def: ModuleDef = raw_def.try_into()?; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index e2f9aae81db..2dc5522f884 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2,9 +2,12 @@ use super::{ ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerId, ReducerOutcome, Scheduler, }; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::{check_hosted_admission, InvocationCaller, SqlCallAuth}; use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit, PublishResult as DeploymentPublishResult}; use crate::db::relational_db::{RelationalDB, Tx}; use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; @@ -73,11 +76,16 @@ use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; use std::num::NonZeroUsize; -use std::sync::atomic::AtomicBool; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore}; +#[cfg(test)] +mod drain_tests; +mod operations; +use operations::ModuleOperations; +pub(in crate::host) use operations::OperationLease; + #[derive(Debug, Default, Clone, From)] pub struct DatabaseUpdate { pub tables: SmallVec<[DatabaseTableUpdate; 1]>, @@ -472,6 +480,7 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl FnOnce(A, &mut ModuleInstance) + Send + 'static, ) where @@ -479,6 +488,7 @@ impl WasmtimeModuleHost { { let label = label.to_owned(); self.executor.enqueue_sync_job(move |state| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("wasm main operation {label} panicked"); on_panic(); @@ -496,15 +506,20 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) + Send + 'static, ) where A: Send + 'static, { let instance_manager = self.procedure_instances.clone(); - let ModuleInstanceLease { instance, slot } = instance_manager.get_instance().await; + let ModuleInstanceLease { instance, slot } = instance_manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); let label = label.to_owned(); self.executor.enqueue_async_job(async move || { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("wasm procedure {label} panicked"); on_panic(); @@ -527,7 +542,8 @@ struct V8ModuleHost { /// A module; used as a bound on `InstanceManager`. trait GenericModule { type Instance: GenericModuleInstance; - async fn create_instance(&self) -> Self::Instance; + type CreationError; + async fn create_instance(&self, operation: Option) -> Result; fn host_type(&self) -> HostType; } @@ -557,8 +573,10 @@ impl GenericModuleInstance for Box { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -567,8 +585,10 @@ impl GenericModule for Arc { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -577,8 +597,9 @@ impl GenericModule for Arc { impl GenericModule for super::v8::JsModule { type Instance = super::v8::JsProcedureInstance; - async fn create_instance(&self) -> Self::Instance { - self.create_instance().await + type CreationError = anyhow::Error; + async fn create_instance(&self, operation: Option) -> Result { + self.create_instance_for_operation(operation).await } fn host_type(&self) -> HostType { HostType::Js @@ -607,15 +628,19 @@ fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, environment: std::collections::BTreeMap, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> (anyhow::Result, bool) { extract_trapped(init_database_inner( replica_ctx, module_def, + module_hash, program, environment, + deployment, call_reducer, )) } @@ -623,10 +648,16 @@ pub(crate) fn init_database( fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, environment: std::collections::BTreeMap, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> anyhow::Result<(InitDatabaseResult, bool)> { + anyhow::ensure!( + module_hash == program.hash && spacetimedb_lib::hash_bytes(&program.bytes) == program.hash, + "program does not match the instantiated module" + ); log::debug!("init database"); let timestamp = Timestamp::now(); let stdb = replica_ctx.relational_db(); @@ -635,6 +666,34 @@ fn init_database_inner( let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let auth_ctx = AuthCtx::for_current(owner_identity); + let (tx, admission) = stdb.with_auto_rollback(tx, |tx| { + if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, module_def)?; + // New databases install the bootstrap fence, schema, program and + // receipt in this one transaction. An interrupted initialization + // cannot leave a successful partial deployment behind. + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::ST_MODULE_ID; + if tx.iter(ST_MODULE_ID)?.next().is_none() { + deployment::install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + } + deployment::check_deployment_commit(tx, request, timestamp, &Default::default()) + } else { + deployment::require_unmanaged_publication(tx)?; + Ok(CommitAdmission::Ready) + } + })?; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok(( + InitDatabaseResult { + reducer: None, + tx_offset: from_tx_offset(offset), + }, + false, + )); + } let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { // Create all in-memory tables defined by the module (including submodules), @@ -685,6 +744,10 @@ fn init_database_inner( crate::db::environment::replace(stdb, tx, module_def.environment(), &environment)?; stdb.set_initialized(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + anyhow::Ok(()) }) .inspect_err(|e| log::error!("{e:?}"))?; @@ -750,6 +813,13 @@ pub fn call_identity_connected( stdb.report_mut_tx_metrics(reducer_name, metrics, None); }); + let caller = InvocationCaller::from(&caller_auth); + let flags = caller + .flags_for(module.database_identity, &module.module_def) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + check_hosted_admission(&*mut_tx, stdb, caller.hosted.as_deref()) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + mut_tx .insert_st_client( caller_auth.claims.identity, @@ -759,13 +829,23 @@ pub fn call_identity_connected( .map_err(DBError::from) .map_err(Box::new)?; + if caller.hosted.is_some() { + crate::db::deployment::record_connection_auth( + &mut mut_tx, + caller_connection_id, + caller_auth.claims.identity, + flags, + ) + .map_err(|err| ClientConnectedError::DBError(Box::new(DBError::Other(err.into()))))?; + } + if let Some((reducer_id, reducer_def)) = reducer_lookup { // The module defined a lifecycle reducer to handle new connections. // Call this reducer. // If the call fails (as in, something unexpectedly goes wrong with guest execution), // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); - let params = ModuleHost::call_reducer_params( + let mut params = ModuleHost::call_reducer_params( &module.module_def, caller_auth.claims.identity, Some(caller_connection_id), @@ -777,6 +857,8 @@ pub fn call_identity_connected( FunctionArgs::Nullary, ) .map_err(ReducerCallError::from)?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; let (reducer_outcome, trapped) = call_reducer(tx, params); *trapped_slot = trapped; @@ -821,7 +903,9 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + /// Verified invocation authority. Bit 0 is internal; never client-decoded. pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -843,6 +927,7 @@ impl CallReducerParams { caller_identity, caller_connection_id: ConnectionId::ZERO, call_auth_flags: 1, + hosted_auth: None, client: None, request_id: None, timer: None, @@ -1048,7 +1133,7 @@ impl ViewCommandErrorTarget { pub(in crate::host) struct SqlCommand { pub(in crate::host) db: Arc, pub(in crate::host) sql_text: String, - pub(in crate::host) auth: AuthCtx, + pub(in crate::host) auth: SqlCallAuth, pub(in crate::host) subs: Option, } @@ -1223,6 +1308,7 @@ pub struct CallProcedureParams { pub caller_identity: Identity, pub caller_connection_id: ConnectionId, pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub timer: Option, pub procedure_id: ProcedureId, pub args: ArgsTuple, @@ -1242,6 +1328,7 @@ impl CallProcedureParams { caller_identity, caller_connection_id: ConnectionId::ZERO, call_auth_flags: 1, + hosted_auth: None, timer: None, procedure_id, args, @@ -1273,7 +1360,7 @@ struct ModuleInstanceManager { struct ModuleInstanceLease { instance: I, - slot: Option, + slot: Option>, } /// Holds the single shared instance used by the JS main execution path. @@ -1446,22 +1533,18 @@ impl ModuleInstanceManager { } } - async fn with_instance(&self, f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance)) -> R { - let ModuleInstanceLease { instance, slot } = self.get_instance().await; - let (res, instance) = f(instance).await; - self.return_instance(ModuleInstanceLease { instance, slot }); - res - } - - async fn get_instance(&self) -> ModuleInstanceLease { + async fn get_instance( + &self, + operation: Option, + ) -> Result, M::CreationError> { let slot = if let Some(instance_slots) = &self.instance_slots { - Some( + Some(Arc::new( instance_slots .clone() .acquire_owned() .await .expect("module instance slot semaphore should not close"), - ) + )) } else { None }; @@ -1474,13 +1557,16 @@ impl ModuleInstanceManager { instance } else { let start_time = std::time::Instant::now(); - let res = self.module.create_instance().await; + let res = self + .module + .create_instance(operation.map(|operation| operation.with_pool_slot(slot.clone()))) + .await?; let elapsed_time = start_time.elapsed(); self.metrics.observe_instance_created(elapsed_time); res }; - ModuleInstanceLease { instance, slot } + Ok(ModuleInstanceLease { instance, slot }) } fn return_instance(&self, lease: ModuleInstanceLease) { @@ -1519,10 +1605,10 @@ pub struct ModuleHost { /// Called whenever a reducer call on this host panics. on_panic: Arc, - /// Marks whether this module has been closed by [`Self::exit`]. - /// - /// When this is true, most operations will fail with [`NoSuchModule`]. - closed: Arc, + /// Shared admission and physical-operation drainage for this module. + /// [`Self::exit`] closes admission, rejects new work with [`NoSuchModule`], + /// and waits for accepted executor jobs, including cancelled callers. + operations: Arc, } impl fmt::Debug for ModuleHost { @@ -1538,12 +1624,19 @@ pub struct WeakModuleHost { info: Arc, inner: Weak, on_panic: Weak, - closed: Weak, + operations: Weak, } #[derive(Debug)] pub enum UpdateDatabaseResult { NoUpdateNeeded, + /// A prior successful commit is returned without replacing the current + /// module, which may already be newer than this retried publication. + DeploymentAlreadyCommitted { + result: DeploymentPublishResult, + tx_offset: TransactionOffset, + durable_offset: Option, + }, UpdatePerformed { /// The transaction offset of the successful database update. tx_offset: TransactionOffset, @@ -1568,6 +1661,7 @@ impl UpdateDatabaseResult { self, UpdateDatabaseResult::UpdatePerformed { .. } | UpdateDatabaseResult::NoUpdateNeeded + | UpdateDatabaseResult::DeploymentAlreadyCommitted { .. } | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } ) } @@ -1577,6 +1671,45 @@ impl UpdateDatabaseResult { #[error("no such module")] pub struct NoSuchModule; +#[derive(thiserror::Error, Debug)] +enum PooledCallError { + #[error(transparent)] + NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + Startup(anyhow::Error), +} + +impl From for ProcedureCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for HttpHandlerCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for CallScheduledFunctionError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => Self::InstanceStartup(error), + } + } +} + #[derive(thiserror::Error, Debug)] pub enum ReducerCallError { #[error(transparent)] @@ -1835,7 +1968,7 @@ impl ModuleHost { info, inner, on_panic, - closed: Arc::new(AtomicBool::new(false)), + operations: Arc::new(ModuleOperations::default()), } } @@ -1854,20 +1987,6 @@ impl ModuleHost { matches!(&*self.inner, ModuleHostInner::Js(_)) } - fn is_marked_closed(&self) -> bool { - // `self.closed` isn't used for any synchronization, it's just a shared flag, - // so `Ordering::Relaxed` is sufficient. - self.closed.load(std::sync::atomic::Ordering::Relaxed) - } - - fn guard_closed(&self) -> Result<(), NoSuchModule> { - if self.is_marked_closed() { - Err(NoSuchModule) - } else { - Ok(()) - } - } - fn start_call_timer(&self, label: &str) -> CallTimerGuard { // Record the time until our function starts running. let queue_timer = WORKER_METRICS @@ -1905,7 +2024,7 @@ impl ModuleHost { R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1918,6 +2037,7 @@ impl ModuleHost { let executor = host.executor.clone(); executor .run_sync_job(move |state| { + let _operation = operation; state.with_instance(move |inst| { drop(timer_guard); wasm(arg, inst) @@ -1928,7 +2048,7 @@ impl ModuleHost { ModuleHostInner::Js(host) => { drop(timer_guard); host.main_instance - .with_instance(|inst| async move { js(arg, &inst).await }) + .with_instance(|inst| async move { js(arg, &inst.with_operation(operation)).await }) .await } }) @@ -1944,12 +2064,12 @@ impl ModuleHost { arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) -> R + Send + 'static, js: impl AsyncFnOnce(A, &JsProcedureInstance) -> R, - ) -> Result + ) -> Result where R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1960,27 +2080,35 @@ impl ModuleHost { Ok(match &*self.inner { ModuleHostInner::Wasm(host) => { let executor = host.executor.clone(); - let instance_manager = host.procedure_instances.clone(); - instance_manager - .with_instance(async move |mut inst| { - executor - .run_async_job(async move || { - drop(timer_guard); - let res = wasm(arg, &mut inst).await; - (res, inst) - }) - .await + let manager = host.procedure_instances.clone(); + let ModuleInstanceLease { mut instance, slot } = manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); + let operation = operation.with_pool_slot(slot); + executor + .run_async_job(async move || { + let _operation = operation; + drop(timer_guard); + let result = wasm(arg, &mut instance).await; + manager.return_instance(ModuleInstanceLease { instance, slot: None }); + result }) .await } ModuleHostInner::Js(host) => { - host.procedure_instances - .with_instance(async |inst| { - drop(timer_guard); - let res = js(arg, &inst).await; - (res, inst) - }) + let mut lease = host + .procedure_instances + .get_instance(Some(operation.clone())) .await + .map_err(PooledCallError::Startup)?; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + drop(timer_guard); + let result = js(arg, &lease.instance).await; + self.return_js_procedure_instance(lease); + result } }) } @@ -1991,7 +2119,7 @@ impl ModuleHost { label: &str, arg: A, js: impl FnOnce(A, JsMainInstance, JsFatalHook) -> JsFut, - wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard) -> Result<(), NoSuchModule>, + wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard, OperationLease) -> Result<(), NoSuchModule>, ) -> Result<(), NoSuchModule> where A: Send + 'static, @@ -2003,21 +2131,20 @@ impl ModuleHost { (self.on_panic)(); }); + let operation = self.operations.begin()?; match &*self.inner { ModuleHostInner::Js(js_host) => { - self.guard_closed()?; let on_panic = self.on_panic.clone(); js_host .main_instance - .with_instance(|inst| js(arg, inst, on_panic)) + .with_instance(|inst| js(arg, inst.with_operation(operation), on_panic)) .await; Ok(()) } ModuleHostInner::Wasm(wasm_host) => { - self.guard_closed()?; let timer_guard = self.start_call_timer(label); let on_panic = self.on_panic.clone(); - wasm(arg, wasm_host, on_panic, timer_guard) + wasm(arg, wasm_host, on_panic, timer_guard, operation) } } } @@ -2034,12 +2161,13 @@ impl ModuleHost { label, (cmd, metric), |(cmd, metric), inst, on_panic| async move { inst.enqueue_call_view(cmd, metric, on_panic).await }, - move |(cmd, metric), wasm_host, on_panic, timer_guard| { + move |(cmd, metric), wasm_host, on_panic, timer_guard, operation| { let info = wasm_host.module.info(); wasm_host.enqueue_with_main_instance( label, on_panic, timer_guard, + operation, (cmd, metric), move |(cmd, metric), inst| { let result = inst.call_view(cmd); @@ -2139,7 +2267,7 @@ impl ModuleHost { } /// Invokes the `client_disconnected` reducer, if present, - /// then deletes the client’s rows from `st_client` and `st_connection_credentials`. + /// then deletes the client's rows from `st_client`, `st_connection_credentials`, and `st_connection_auth`. /// If the reducer fails, the rows are still deleted. /// Calling this on an already-disconnected client is a no-op. pub fn call_identity_disconnected_inner( @@ -2205,22 +2333,35 @@ impl ModuleHost { // The module defined a lifecycle reducer to handle disconnects. Call it. // If it succeeds, `WasmModuleInstance::call_reducer_with_tx` has already ensured // that `st_client` is updated appropriately. + let flags = crate::db::deployment::connection_auth_flags(&mut_tx, caller_connection_id, caller_identity) + .map_err(|err| { + InvalidReducerArguments(InvalidFunctionArguments { + err: err.into(), + function_name: reducer_name.clone().into(), + }) + }); let tx = Some(mut_tx); - let result = Self::call_reducer_params( - &info.module_def, - caller_identity, - Some(caller_connection_id), - None, - None, - None, - reducer_id, - reducer_def, - FunctionArgs::Nullary, - ) - .map(|params| { - let (res, trapped) = call_reducer(tx, params); - *trapped_slot = trapped; - res + let result = flags.and_then(|flags| { + Self::call_reducer_params( + &info.module_def, + caller_identity, + Some(caller_connection_id), + None, + None, + None, + reducer_id, + reducer_def, + FunctionArgs::Nullary, + ) + .map(|mut params| { + // This host event retains the connection's captured authority, + // including after credential expiry, revocation, or host recovery. + // It must not carry a live hosted proof that could block cleanup. + params.call_auth_flags = flags; + let (res, trapped) = call_reducer(tx, params); + *trapped_slot = trapped; + res + }) }); // If it failed, we still need to update `st_client`: the client's not coming back. @@ -2313,6 +2454,7 @@ impl ModuleHost { caller_identity, caller_connection_id, call_auth_flags: 0, + hosted_auth: None, client, request_id, timer, @@ -2323,7 +2465,7 @@ impl ModuleHost { fn reducer_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2331,6 +2473,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ReducerCallError::NoSuchReducer)?; + let caller_identity = caller.identity; let (reducer_id, reducer_def, owning_def) = self .info .module_def @@ -2342,25 +2488,25 @@ impl ModuleHost { if !reducer_def .visibility - .allows_invocation(false, self.is_database_owner(caller_identity)) + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) { return Err(ReducerCallError::NoSuchReducer); } - Ok(( + let mut params = Self::call_reducer_params( + owning_def, + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, reducer_def, - Self::call_reducer_params( - owning_def, - caller_identity, - caller_connection_id, - client, - request_id, - timer, - reducer_id, - reducer_def, - args, - )?, - )) + args, + )?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; + Ok((reducer_def, params)) } async fn call_reducer_with_params( @@ -2388,7 +2534,7 @@ impl ModuleHost { async fn with_reducer_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2399,7 +2545,7 @@ impl ModuleHost { ) -> Result { let res = async { let (reducer_def, params) = self.reducer_call_params( - caller_identity, + caller, caller_connection_id, client, request_id, @@ -2424,7 +2570,7 @@ impl ModuleHost { pub async fn call_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2433,7 +2579,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2447,7 +2593,7 @@ impl ModuleHost { pub async fn enqueue_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2456,7 +2602,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result<(), ReducerCallError> { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2470,11 +2616,12 @@ impl ModuleHost { reducer_name, call.params, |params, inst, on_panic| async move { inst.enqueue_reducer(params, on_panic).await }, - move |params, wasm_host, on_panic, timer_guard| { + move |params, wasm_host, on_panic, timer_guard, operation| { wasm_host.enqueue_with_main_instance( &reducer_label, on_panic, timer_guard, + operation, params, move |params, inst| { let _ = inst.call_reducer(params); @@ -2599,10 +2746,15 @@ impl ModuleHost { &self, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + InvocationCaller { + identity: auth.caller(), + hosted: auth.hosted.clone(), + } + .flags_for(self.info.database_identity, &self.info.module_def)?; let cmd = SqlCommand { db, sql_text, @@ -2618,18 +2770,15 @@ impl ModuleHost { pub async fn call_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> CallProcedureReturn { let res = async { - let call = - self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args)?; - self.call_procedure_with_params(&call.name, call.params) - .await - .map_err(Into::into) + let call = self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args)?; + self.call_procedure_with_params(&call.name, call.params).await } .await; @@ -2642,7 +2791,7 @@ impl ModuleHost { pub(crate) async fn enqueue_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, @@ -2650,7 +2799,7 @@ impl ModuleHost { target: ProcedureResultTarget, ) -> Result<(), BroadcastError> { let PreparedProcedureCall { name, params } = - match self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args) { + match self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args) { Ok(value) => value, Err(err) => { return self.send_procedure_error(procedure_name, timer, target, err); @@ -2664,14 +2813,39 @@ impl ModuleHost { (self.on_panic)(); }); - if let Err(err) = self.guard_closed() { - return self.send_procedure_error(&procedure_name, timer, target, err.into()); - } + let operation = match self.operations.begin() { + Ok(operation) => operation, + Err(err) => return self.send_procedure_error(&procedure_name, timer, target, err.into()), + }; match &*self.inner { ModuleHostInner::Js(host) => { - let lease = host.procedure_instances.get_instance().await; - let call = lease.instance.enqueue_procedure(params).await; + let mut lease = match host.procedure_instances.get_instance(Some(operation.clone())).await { + Ok(lease) => lease, + Err(error) => { + return self.send_procedure_error( + &procedure_name, + timer, + target, + PooledCallError::Startup(error).into(), + ); + } + }; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + let call = match lease.instance.enqueue_procedure(params).await { + Ok(call) => call, + Err(error) => { + self.return_js_procedure_instance(lease); + return self.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ); + } + }; let module = self.clone(); tokio::spawn(async move { match call.receive().await { @@ -2681,6 +2855,16 @@ impl ModuleHost { log::warn!("failed to send procedure result: {err:#}"); } } + JsProcedureCallCompletion::StartupFailed(error) => { + if let Err(error) = module.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ) { + log::warn!("failed to send procedure startup error: {error:#}"); + } + } JsProcedureCallCompletion::Panicked | JsProcedureCallCompletion::WorkerExited => { log::error!("detached JS procedure worker failed before returning a result"); (module.on_panic)(); @@ -2701,6 +2885,7 @@ impl ModuleHost { &procedure_name, on_panic, timer_guard, + operation, params, async move |params, inst| { let ret = inst.call_procedure(params).await; @@ -2720,10 +2905,11 @@ impl ModuleHost { } } - fn return_js_procedure_instance(&self, lease: ModuleInstanceLease) { + fn return_js_procedure_instance(&self, mut lease: ModuleInstanceLease) { let ModuleHostInner::Js(host) = &*self.inner else { return; }; + lease.instance.set_operation(None); host.procedure_instances.return_instance(lease); } @@ -2825,14 +3011,14 @@ impl ModuleHost { fn prepare_procedure_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result { let (procedure_def, params) = - self.procedure_call_params(caller_identity, caller_connection_id, timer, procedure_name, args)?; + self.procedure_call_params(caller, caller_connection_id, timer, procedure_name, args)?; Ok(PreparedProcedureCall { name: procedure_def.name.to_string(), params, @@ -2841,12 +3027,16 @@ impl ModuleHost { fn procedure_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ProcedureCallError::NoSuchProcedure)?; + let caller_identity = caller.identity; let (procedure_id, procedure_def, owning_def) = self .info .module_def @@ -2855,7 +3045,7 @@ impl ModuleHost { if !procedure_def .visibility - .allows_invocation(false, self.is_database_owner(caller_identity)) + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) { return Err(ProcedureCallError::NoSuchProcedure); } @@ -2871,7 +3061,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, - call_auth_flags: 0, + call_auth_flags: flags, + hosted_auth: caller.hosted, timer, procedure_id, args, @@ -2888,7 +3079,7 @@ impl ModuleHost { &self, name: &str, params: CallProcedureParams, - ) -> Result { + ) -> Result { call_pooled_instance!( self, name, @@ -2896,6 +3087,7 @@ impl ModuleHost { |params, inst| inst.call_procedure(params).await, |params, inst| inst.call_procedure(params).await, ) + .map_err(Into::into) } pub async fn call_http_handler( @@ -2946,10 +3138,13 @@ impl ModuleHost { self, "scheduled procedure", params, - |params, inst| inst.call_scheduled_procedure(params).await, - |params, inst| inst.call_scheduled_procedure(params).await, + |params, inst| Ok(inst.call_scheduled_procedure(params).await), + |params, inst| inst + .call_scheduled_procedure(params) + .await + .map_err(|error| CallScheduledFunctionError::InstanceStartup(error.into())), ) - .map_err(Into::into) + .map_err(CallScheduledFunctionError::from)? } /// Materializes the views return by the `view_collector`, if not already materialized, @@ -3210,14 +3405,58 @@ impl ModuleHost { program: Program, environment: std::collections::BTreeMap, ) -> Result { - call_instance!( + self.init_database_with_environment_and_deployment(program, environment, None) + .await + } + + pub async fn init_database_with_deployment( + &self, + program: Program, + deployment: Option, + ) -> Result { + self.init_database_with_environment_and_deployment(program, Default::default(), deployment) + .await + } + + /// Initialize the complete environment, bootstrap fence, schema and receipt + /// atomically, then confirm deployment durability before activation. + pub async fn init_database_with_environment_and_deployment( + &self, + program: Program, + environment: std::collections::BTreeMap, + deployment: Option, + ) -> Result { + let confirm_deployment = deployment.is_some(); + let result = call_instance!( self, "", - (program, environment), - |(program, environment), inst| inst.init_database(program, environment), - |(program, environment), inst| inst.init_database(program, environment).await, + (program, environment, deployment), + |(p, e, d), inst| inst.init_database(p, e, d), + |(p, e, d), inst| inst.init_database(p, e, d).await, )? - .map_err(InitDatabaseError::Other) + .map_err(InitDatabaseError::Other)?; + if confirm_deployment + && result.reducer.as_ref().is_none_or(|result| result.is_ok()) + && let Some(mut durability) = self.relational_db().durable_tx_offset() + { + // A read barrier after init includes its receipt even when the + // module has no init reducer. Do not block the async worker on + // the datastore lock while capturing that barrier. + let db = self.relational_db().clone(); + let offset = tokio::task::spawn_blocking(move || { + let tx = db.begin_tx(Workload::Internal); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + offset + }) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + durability + .wait_for(offset) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + } + Ok(result) } pub async fn update_database( @@ -3236,25 +3475,58 @@ impl ModuleHost { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + ) -> Result { + self.update_database_with_environment_and_deployment(program, old_module_info, policy, environment, None) + .await + } + + pub async fn update_database_with_deployment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + deployment: Option, + ) -> Result { + self.update_database_with_environment_and_deployment( + program, + old_module_info, + policy, + Default::default(), + deployment, + ) + .await + } + + /// Commit complete ENV, program migration and the deployment receipt atomically. + pub async fn update_database_with_environment_and_deployment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + environment: std::collections::BTreeMap, + deployment: Option, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy, environment), - |(a, b, c, d), inst| inst.update_database(a, b, c, d), - |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, + (program, old_module_info, policy, environment, deployment), + |(a, b, c, e, d), inst| inst.update_database(a, b, c, e, d), + |(a, b, c, e, d), inst| inst.update_database(a, b, c, e, d).await, )? } pub async fn exit(&self) { - // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. - self.closed.store(true, std::sync::atomic::Ordering::Relaxed); + // Admission and closure share one lock. Already admitted work retains + // its lease inside the physical executor after cancellation of a caller. + self.operations.close(); self.scheduler().close(); self.exited().await; } pub async fn exited(&self) { self.scheduler().closed().await; + self.operations.close(); + self.operations.drained().await; } pub fn inject_logs(&self, log_level: LogLevel, function_name: &str, message: &str) { @@ -3327,11 +3599,12 @@ impl ModuleHost { label, request, |request, inst, on_panic| async move { inst.enqueue_one_off_query(request, on_panic).await }, - move |request, wasm_host, on_panic, timer_guard| { + move |request, wasm_host, on_panic, timer_guard, operation| { let executor = wasm_host.executor.clone(); let info = wasm_host.module.info(); let label = label.to_owned(); executor.enqueue_sync_job(move |_| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("websocket one-off query operation {label} panicked"); on_panic(); @@ -3484,8 +3757,10 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { - ws_v1::OneOffTable { table_name, rows } + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { + Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { + ws_v1::OneOffTable { table_name, rows } + }) }); let total_host_execution_duration = timer.elapsed().into(); @@ -3563,10 +3838,11 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query::(&db, &tx, &auth, &query, &rlb_pool, |table, rows| { ws_v2::SingleTableRows { table, rows } - }); + }) + }); let (message, metrics) = match result { Ok((rows, metrics)) => { @@ -3604,6 +3880,7 @@ impl ModuleHost { /// for tables without primary keys. It is only used in the benchmarks. /// Note: this doesn't drop the table, it just clears it! pub fn clear_table(&self, table_name: &str) -> Result<(), anyhow::Error> { + let _operation = self.operations.begin()?; let db = self.relational_db(); db.with_auto_commit(Workload::Internal, |tx| { @@ -3625,7 +3902,7 @@ impl ModuleHost { info: self.info.clone(), inner: Arc::downgrade(&self.inner), on_panic: Arc::downgrade(&self.on_panic), - closed: Arc::downgrade(&self.closed), + operations: Arc::downgrade(&self.operations), } } @@ -3664,12 +3941,12 @@ impl WeakModuleHost { pub fn upgrade(&self) -> Option { let inner = self.inner.upgrade()?; let on_panic = self.on_panic.upgrade()?; - let closed = self.closed.upgrade()?; + let operations = self.operations.upgrade()?; Some(ModuleHost { info: self.info.clone(), inner, on_panic, - closed, + operations, }) } } diff --git a/crates/core/src/host/module_host/drain_tests.rs b/crates/core/src/host/module_host/drain_tests.rs new file mode 100644 index 00000000000..f1f98d5fba8 --- /dev/null +++ b/crates/core/src/host/module_host/drain_tests.rs @@ -0,0 +1,412 @@ +//! Actual local database writers, Wasm/JS worker queues and cancelled callers. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::host_controller::{HostController, HostRuntimeConfig}; +use crate::util::jobs::JobCores; +use spacetimedb_auth::identity::SpacetimeIdentityClaims; +use spacetimedb_datastore::system_tables::ModuleKind; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; +use std::collections::BTreeMap; +use tokio::time::timeout; + +fn javascript() -> Program { + let mut definition = RawModuleDefV10Builder::new(); + definition.add_lifecycle_reducer( + Lifecycle::OnDisconnect, + "disconnected", + spacetimedb_sats::ProductType::unit(), + ); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(definition.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks}} from "spacetime:sys@1.0"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), + __call_reducer__: () => ({{tag:"ok"}}) }}); + "# + ) + .into_bytes(), + ) +} + +fn fixture(id: u64, program: Program) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let controller = HostController::new( + data.clone(), + crate::db::Config { + storage: crate::db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }), + Arc::new(crate::energy::NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: if program.kind == ModuleKind::JS { + HostType::Js + } else { + HostType::Wasm + }, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database) +} + +async fn until(mut condition: impl FnMut() -> bool) { + timeout(Duration::from_secs(5), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +fn client_auth() -> ConnectionAuthCtx { + SpacetimeIdentityClaims { + identity: Identity::ONE, + subject: "local-test".into(), + issuer: "local-test".into(), + audience: Box::new([]), + iat: std::time::SystemTime::now(), + exp: None, + extra: None, + } + .try_into() + .unwrap() +} + +async fn queued_disconnect(js: bool, cancel: bool, id: u64) { + let program = if js { + javascript() + } else { + crate::host::empty_module::program(crate::host::empty_module::VERSION).unwrap() + }; + let (_directory, controller, database) = fixture(id, program); + let module = controller + .get_or_launch_module_host(database.clone(), id) + .await + .unwrap(); + let client = ClientActorId::for_test(Identity::ONE); + module + .call_identity_connected(client_auth(), client.connection_id) + .await + .unwrap(); + let db = module.relational_db().clone(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_some()); + let call = tokio::spawn({ + let module = module.clone(); + async move { module.disconnect_client(client).await } + }); + until(|| module.operations.active() == 1).await; + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + if cancel { + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + } else { + drop(call); + } + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + let _ = db.rollback_mut_tx(tx); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_none()); + let _ = db.rollback_mut_tx(tx); + assert!(module.clear_all_clients().await.is_err()); + let result = crate::sql::execute::run( + db.clone(), + "SELECT * FROM st_client".into(), + AuthCtx::new(db.owner_identity(), db.owner_identity()), + None, + None, + &mut Vec::new(), + ) + .await; + assert!(matches!(result, Err(DBError::DatabaseClosed))); + assert!(controller.get_module_host(id).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_queued_disconnect_survives_cancellation() { + queued_disconnect(false, true, 0xd100).await; + queued_disconnect(false, false, 0xd101).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_queued_disconnect_survives_cancellation() { + queued_disconnect(true, true, 0xd102).await; + queued_disconnect(true, false, 0xd103).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_procedure_owns_external_wait_and_late_commit() { + let id = 0xd104; + let schema = spacetimedb_lib::environment::EnvironmentSchema::new(vec![ + spacetimedb_lib::environment::EnvironmentDeclaration { + name: "AFTER_IO".into(), + constraint: spacetimedb_lib::environment::EnvironmentConstraint::AnyString, + optional: true, + }, + ]) + .unwrap(); + let (_directory, controller, database) = fixture(id, crate::host::empty_module::declared_program(&schema).unwrap()); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let call = tokio::spawn({ + let module = module.clone(); + let db = db.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + module + .call_pooled( + "external-wait-test", + (), + async move |_, _| { + started.add_permits(1); + release.acquire().await.unwrap().forget(); + db.with_auto_commit(Workload::Internal, |tx| { + crate::db::environment::replace( + &db, + tx, + &schema, + &BTreeMap::from([("AFTER_IO".into(), "committed".into())]), + ) + .map_err(anyhow::Error::from) + }) + .unwrap(); + }, + async |_, _| unreachable!(), + ) + .await + .unwrap(); + } + }); + timeout(Duration::from_secs(5), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_tx(Workload::Internal); + assert_eq!( + crate::db::environment::get(&tx, "AFTER_IO").unwrap().as_deref(), + Some("committed") + ); + let _ = db.release_tx(tx); +} + +/// Opt in only for the existing loopback test feature. Production IP filtering +/// stays enabled. Run with proxy variables cleared, as asserted below. +#[cfg(feature = "allow_loopback_http_for_tests")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_procedure_owns_actual_http_until_late_commit() { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for variable in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert!( + std::env::var_os(variable).is_none(), + "clear {variable} for this disposable loopback test" + ); + } + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + assert!(address.ip().is_loopback()); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let server = tokio::spawn({ + let started = started.clone(); + let release = release.clone(); + async move { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + let mut request = [0u8; 4096]; + assert!(socket.read(&mut request).await.unwrap() > 0); + started.add_permits(1); + release.acquire().await.unwrap().forget(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + } + }); + let request = bsatn::to_vec(&spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: format!("http://{address}/owned-test"), + version: spacetimedb_lib::http::Version::Http11, + }) + .unwrap(); + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes(ModuleKind::JS, format!(r#" + import {{register_hooks,table_id_from_name,datastore_insert_bsatn}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures,procedure_http_request,procedure_start_mut_tx,procedure_commit_mut_tx}} from "spacetime:sys@1.2"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => {{ + procedure_http_request(new Uint8Array({request:?}), ""); + procedure_start_mut_tx(); + datastore_insert_bsatn(table_id_from_name("rows"), new Uint8Array([42,0,0,0,0,0,0,0])); + procedure_commit_mut_tx(); + return new Uint8Array(8); + }} }}); + "#).into_bytes()); + let id = 0xd105; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(10), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(10)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(10), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + server.await.unwrap(); + let tx = db.begin_tx(Workload::Internal); + let table = db.table_id_from_name(&tx, "rows").unwrap().unwrap(); + assert_eq!(tx.table_row_count(table), Some(1)); + let _ = db.release_tx(tx); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_cancelled_js_procedure_startup_retains_physical_slot() { + use futures::StreamExt; + let mut schema = RawModuleDefV10Builder::new(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks,console_log}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures}} from "spacetime:sys@1.2"; + function startup() {{ console_log(2, "physical-startup-entered"); }} + startup(); + const until = Date.now() + 2000; + while (Date.now() < until) {{}} + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => new Uint8Array(8)}}); + "# + ) + .into_bytes(), + ); + let id = 0xd106; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let ModuleHostInner::Js(host) = &*module.inner else { + unreachable!() + }; + let slots = host.procedure_instances.instance_slots.as_ref().unwrap(); + let maximum = slots.available_permits(); + let mut logs = module.database_logger().tail(Some(0), true).await.unwrap(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(5), async { + loop { + let log = logs.next().await.unwrap().unwrap(); + if String::from_utf8_lossy(&log).contains("physical-startup-entered") { + break; + } + } + }) + .await + .unwrap(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert_eq!(module.operations.active(), 1); + assert_eq!(slots.available_permits(), maximum - 1); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert!(!closing.is_finished()); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(slots.available_permits(), maximum); +} diff --git a/crates/core/src/host/module_host/operations.rs b/crates/core/src/host/module_host/operations.rs new file mode 100644 index 00000000000..4906f304a5c --- /dev/null +++ b/crates/core/src/host/module_host/operations.rs @@ -0,0 +1,126 @@ +//! Physical operation ownership, independent of an HTTP/WebSocket waiter's lifetime. +use super::NoSuchModule; +use parking_lot::Mutex; +use std::sync::Arc; +use tokio::sync::{Notify, OwnedSemaphorePermit}; + +#[derive(Default)] +pub(super) struct ModuleOperations { + state: Mutex, + drained: Notify, +} + +#[derive(Default)] +struct State { + closed: bool, + active: usize, +} + +impl ModuleOperations { + pub(super) fn begin(self: &Arc) -> Result { + let mut state = self.state.lock(); + if state.closed { + return Err(NoSuchModule); + } + state.active = state.active.checked_add(1).ok_or(NoSuchModule)?; + Ok(OperationLease { + _token: Arc::new(ActiveOperation(self.clone())), + pool_slot: None, + }) + } + + #[cfg(test)] + pub(super) fn active(&self) -> usize { + self.state.lock().active + } + + #[cfg(test)] + pub(super) fn is_closed(&self) -> bool { + self.state.lock().closed + } + + pub(super) fn close(&self) { + self.state.lock().closed = true; + } + + pub(super) async fn drained(&self) { + loop { + let notified = self.drained.notified(); + tokio::pin!(notified); + // Register before reading the counter, including on a multi-threaded runtime. + notified.as_mut().enable(); + if self.state.lock().active == 0 { + return; + } + notified.await; + } + } +} + +/// Clones describe one admitted operation, not additional admissions. A clone +/// moves into the physical job/request before it is enqueued. Cancellation of +/// the caller therefore cannot release admission or a pooled instance's slot. +#[derive(Clone)] +pub(in crate::host) struct OperationLease { + _token: Arc, + pool_slot: Option>, +} + +impl OperationLease { + pub(super) fn with_pool_slot(mut self, slot: Option>) -> Self { + self.pool_slot = slot; + self + } +} + +struct ActiveOperation(Arc); + +impl Drop for ActiveOperation { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn close_counts_previously_admitted_work_even_before_enqueue() { + let operations = Arc::new(ModuleOperations::default()); + let operation = operations.begin().unwrap(); + operations.close(); + assert!(operations.begin().is_err()); + let waiter = tokio::spawn({ + let operations = operations.clone(); + async move { operations.drained().await } + }); + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + drop(operation); + waiter.await.unwrap(); + operations.drained().await; + } + + #[tokio::test] + async fn cancelled_waiter_does_not_release_physical_job_or_pool_slot() { + let operations = Arc::new(ModuleOperations::default()); + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let caller = operations + .begin() + .unwrap() + .with_pool_slot(Some(Arc::new(slots.clone().acquire_owned().await.unwrap()))); + let physical_job = caller.clone(); + drop(caller); + operations.close(); + assert_eq!(slots.available_permits(), 0); + assert_eq!(operations.state.lock().active, 1); + drop(physical_job); + operations.drained().await; + assert_eq!(slots.available_permits(), 1); + } +} diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index a07dd43597e..4a8c4005a81 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -321,6 +321,8 @@ impl ScheduledFunctionParams { pub(crate) enum CallScheduledFunctionError { #[error(transparent)] NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + InstanceStartup(anyhow::Error), } #[cfg(target_pointer_width = "64")] @@ -413,6 +415,17 @@ impl SchedulerActor { // If the module already exited, leave the `ScheduledFunction` in // the database for when the module restarts. Err(CallScheduledFunctionError::NoSuchModule(_)) => {} + Err(CallScheduledFunctionError::InstanceStartup(error)) => { + // No transaction or procedure ran. Keep the schedule available + // and retry with a delay, including under shared deadline pressure. + log::warn!("scheduled procedure instance startup failed: {error:#}"); + if startup_retry_is_needed(module_host.info().relational_db(), &item) { + let key = self.queue.insert(item, Duration::from_secs(1)); + if let Some(id) = id { + self.key_map.insert(id, key); + } + } + } Ok(CallScheduledFunctionResult { reschedule: None }) => { // nothing to do } @@ -436,6 +449,26 @@ impl SchedulerActor { } } +/// A cancelled table-backed schedule must not retry forever when every new +/// procedure instance fails before reaching the normal schedule-row lookup. +fn startup_retry_is_needed(db: &RelationalDB, item: &QueueItem) -> bool { + let QueueItem::Id { id, .. } = item else { + return true; + }; + let exists = db.with_read_only(Workload::Internal, |tx| { + db.iter_by_col_eq(tx, id.table_id, id.id_column, &id.schedule_id.into()) + .map(|mut rows| rows.next().is_some()) + }); + match exists { + Ok(exists) => exists, + Err(error) => { + // A failed read is not proof that the user cancelled the schedule. + log::warn!("could not check scheduled procedure startup retry: {error:#}"); + true + } + } +} + #[derive(Debug)] pub(crate) struct CallScheduledFunctionResult { reschedule: Option, @@ -986,3 +1019,61 @@ mod tests { assert_eq!(next, ts(1_400)); } } + +#[cfg(test)] +mod startup_retry_tests { + use super::*; + use crate::db::relational_db::tests_utils::{insert, with_auto_commit, TestDB}; + use spacetimedb_sats::{product, AlgebraicType}; + + fn item(table_id: TableId, schedule_id: u64) -> QueueItem { + QueueItem::Id { + id: ScheduledFunctionId { + table_id, + schedule_id, + id_column: 0.into(), + at_column: 0.into(), + }, + function_name: "task".into(), + at: Timestamp::now(), + } + } + + #[test] + fn execution_deadline_startup_retry_stops_after_schedule_deletion() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + let table = db.create_table_for_test("pending", &[("id", AlgebraicType::U64)], &[0.into()])?; + with_auto_commit(&db, |tx| { + insert(&db, tx, table, &(7u64,))?; + insert(&db, tx, table, &(8u64,))?; + Ok::<_, anyhow::Error>(()) + })?; + + let cancelled = item(table, 7); + assert!(startup_retry_is_needed(&db, &cancelled)); + with_auto_commit(&db, |tx| { + assert_eq!(db.delete_by_rel(tx, table, [product!(7u64)]), 1); + Ok::<_, anyhow::Error>(()) + })?; + + // An unrelated remaining schedule cannot keep the cancelled ID alive. + assert!(!startup_retry_is_needed(&db, &cancelled)); + assert!(startup_retry_is_needed(&db, &item(table, 8))); + Ok(()) + } + + #[test] + fn execution_deadline_startup_retry_preserves_read_errors_and_volatile_calls() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + // The point read fails rather than positively observing a missing row. + assert!(startup_retry_is_needed(&db, &item(u32::MAX.into(), 7))); + assert!(startup_retry_is_needed( + &db, + &QueueItem::VolatileNonatomicImmediate { + function_name: "task".into(), + args: FunctionArgs::Nullary, + }, + )); + Ok(()) + } +} diff --git a/crates/core/src/host/v8/execution_deadline.rs b/crates/core/src/host/v8/execution_deadline.rs new file mode 100644 index 00000000000..ac6b1f3d1c9 --- /dev/null +++ b/crates/core/src/host/v8/execution_deadline.rs @@ -0,0 +1,272 @@ +//! Direct cross-thread termination, without executing a V8 interrupt callback. +//! +//! Pinned v8 145's IsolateHandle is Send+Sync and protects termination with its +//! annex mutex. No scope, raw isolate pointer, Rc or Cell leaves the isolate +//! thread. One persistent timer serves bounded active registrations. Finishing +//! or dropping a guard waits out any in-flight termination before the isolate +//! may reset termination or begin another invocation. + +use std::{ + collections::BTreeMap, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Condvar, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +#[derive(Debug, thiserror::Error)] +#[error("JavaScript execution exceeded its wall-clock limit")] +pub(super) struct ExecutionTimedOut; + +const MAX_ACTIVE_DEADLINES: usize = 16_384; +static ACTIVE_DEADLINES: AtomicUsize = AtomicUsize::new(0); +type Key = (Instant, u64); +static SERVICE: OnceLock, String>> = OnceLock::new(); + +struct Capacity; +impl Capacity { + fn acquire() -> io::Result { + ACTIVE_DEADLINES + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + (count < MAX_ACTIVE_DEADLINES).then_some(count + 1) + }) + .map_err(|_| io::Error::other("JavaScript execution deadline capacity exhausted"))?; + Ok(Self) + } +} +impl Drop for Capacity { + fn drop(&mut self) { + ACTIVE_DEADLINES.fetch_sub(1, Ordering::Relaxed); + } +} + +struct Registration { + expires: Instant, + state: Mutex, +} +struct State { + finished: bool, + expired: bool, + handle: v8::IsolateHandle, +} +#[derive(Default)] +struct Queue { + sequence: u64, + entries: BTreeMap>, +} +struct DeadlineService { + queue: Mutex, + wake: Condvar, +} + +impl DeadlineService { + fn start() -> io::Result> { + let service = Arc::new(Self { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }); + std::thread::Builder::new().name("v8-deadlines".into()).spawn({ + let service = service.clone(); + move || service.run() + })?; + Ok(service) + } + + fn run(&self) { + loop { + let registration = { + let mut queue = self.queue.lock().unwrap_or_else(|error| error.into_inner()); + loop { + let Some((&(expires, _), _)) = queue.entries.first_key_value() else { + queue = self.wake.wait(queue).unwrap_or_else(|error| error.into_inner()); + continue; + }; + let remaining = expires.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break queue.entries.pop_first().unwrap().1; + } + queue = self + .wake + .wait_timeout(queue, remaining) + .unwrap_or_else(|error| error.into_inner()) + .0; + } + }; + // Never hold the queue lock while acquiring a registration lock. + // Cancellation may hold this lock while removing its queued entry. + let mut state = registration.state.lock().unwrap_or_else(|error| error.into_inner()); + if !state.finished { + state.expired = true; + // finish() must acquire this same lock before resetting V8. + state.handle.terminate_execution(); + } + } + } +} + +pub(super) struct ExecutionDeadline { + service: Arc, + registration: Arc, + key: Key, + finished: bool, + // Includes expired calls still returning from a native syscall. The timer + // popping its queue entry never releases this active-call reservation. + _capacity: Capacity, +} + +impl ExecutionDeadline { + pub fn start(handle: v8::IsolateHandle, timeout: Duration) -> io::Result { + let capacity = Capacity::acquire()?; + let service = SERVICE + .get_or_init(|| DeadlineService::start().map_err(|error| error.to_string())) + .as_ref() + .map_err(|error| io::Error::other(error.clone()))? + .clone(); + let expires = Instant::now() + timeout; + let registration = Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle, + }), + }); + let mut queue = service.queue.lock().unwrap_or_else(|error| error.into_inner()); + queue.sequence = queue + .sequence + .checked_add(1) + .ok_or_else(|| io::Error::other("JavaScript execution deadline sequence exhausted"))?; + let key = (expires, queue.sequence); + queue.entries.insert(key, registration.clone()); + drop(queue); + service.wake.notify_one(); + Ok(Self { + service, + registration, + key, + finished: false, + _capacity: capacity, + }) + } + + pub fn finish(mut self) -> bool { + let expired = self.cancel(); + self.finished = true; + expired + } + + fn cancel(&self) -> bool { + let mut state = self + .registration + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + if !state.finished { + // A timer delayed by OS scheduling must still make a late return + // roll back, even if it has not issued its termination request yet. + state.expired |= Instant::now() >= self.registration.expires; + state.finished = true; + } + let expired = state.expired; + // The timer releases the queue lock before taking this lock, so this + // order cannot invert its locks. Remove completed calls immediately, + // rather than accumulating their registrations for 120 seconds. + self.service + .queue + .lock() + .unwrap_or_else(|error| error.into_inner()) + .entries + .remove(&self.key); + drop(state); + self.service.wake.notify_one(); + expired + } +} + +impl Drop for ExecutionDeadline { + fn drop(&mut self) { + if !self.finished { + self.cancel(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::v8::to_value::test::with_scope; + + fn run(scope: &mut v8::PinScope<'_, '_>, source: &str) -> bool { + let source = v8::String::new(scope, source).unwrap(); + v8::Script::compile(scope, source, None).unwrap().run(scope).is_some() + } + + #[test] + fn deadline_terminates_and_isolate_can_run_again() { + with_scope(|scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + assert!(!run(scope, "for (;;) {}")); + assert!(deadline.finish()); + scope.cancel_terminate_execution(); + assert!(run(scope, "1 + 1")); + }); + } + + #[test] + fn finished_and_dropped_registrations_cannot_terminate_later_execution() { + with_scope(|scope| { + for drop_guard in [false, true] { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + if drop_guard { + drop(deadline); + } else { + assert!(!deadline.finish()); + } + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + assert!(run( + scope, + "{ const end = Date.now() + 80; while (Date.now() < end) {} }" + )); + } + for _ in 0..1_000 { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_secs(1)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + assert!(run(scope, "1 + 1")); + assert!(!deadline.finish()); + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + } + }); + } + + #[test] + fn late_return_is_expired_even_before_timer_observes_it() { + with_scope(|scope| { + // A service without a timer deterministically models OS delay. + let expires = Instant::now(); + let deadline = ExecutionDeadline { + service: Arc::new(DeadlineService { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }), + registration: Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle: scope.thread_safe_handle(), + }), + }), + key: (expires, 1), + finished: false, + _capacity: Capacity::acquire().unwrap(), + }; + assert!(deadline.finish()); + assert!(run(scope, "1 + 1")); + }); + } +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 207a4a08ec6..9624a5a4244 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -57,6 +57,7 @@ use self::error::{ catch_exception, exception_already_thrown, log_traceback, ErrorOrException, ExcResult, ExceptionThrown, PinTryCatch, Throwable, }; +use self::execution_deadline::{ExecutionDeadline, ExecutionTimedOut}; use self::ser::serialize_to_js; use self::string::{str_from_ident, IntoJsString}; use self::syscall::{ @@ -73,6 +74,7 @@ use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSen use crate::config::{V8Config, V8HeapPolicyConfig}; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{ChunkPool, InstanceEnv, TxSlot}; +use crate::host::module_host::OperationLease; use crate::host::module_host::{ call_identity_connected, init_database, ClientConnectedError, HttpHandlerCallError, OneOffQueryRequest, SqlCommand, SqlCommandResult, ViewCommand, ViewCommandMetric, ViewCommandResult, @@ -85,6 +87,7 @@ use crate::host::wasm_common::module_host_actor::{ ReducerExecuteResult, ReducerOp, ViewExecuteResult, ViewOp, WasmInstance, }; use crate::host::wasm_common::{RowIters, TimingSpanSet}; +use crate::host::ProcedureCallError; use crate::host::{InitDatabaseResult, ModuleHost, ReducerCallError, ReducerCallResult, Scheduler}; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; @@ -110,8 +113,8 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; -use std::sync::{Arc, LazyLock}; -use std::time::Instant; +use std::sync::{Arc, LazyLock, OnceLock}; +use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; use v8::script_compiler::{compile_module, Source}; use v8::{ @@ -123,6 +126,7 @@ mod budget; mod builtins; mod de; mod error; +mod execution_deadline; mod from_value; mod ser; mod string; @@ -154,6 +158,7 @@ impl V8Runtime { program_bytes: &[u8], core: AllocatedJobCore, ) -> anyhow::Result { + self.config.validate_execution_timeout()?; V8_RUNTIME_GLOBAL .make_actor(mcc, program_bytes, core, self.config) .await @@ -256,6 +261,7 @@ impl V8RuntimeInner { load_balance_guard.clone(), core_pinner.clone(), heap_policy, + config.execution_timeout, metrics.clone(), ) .await?; @@ -266,6 +272,7 @@ impl V8RuntimeInner { core_pinner, procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, + execution_timeout: config.execution_timeout, metrics, }; @@ -281,6 +288,7 @@ pub struct JsModule { core_pinner: CorePinner, procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, } @@ -305,7 +313,10 @@ impl JsModule { self.procedure_instance_pool_size } - async fn create_procedure_instance(&self) -> JsProcedureInstance { + async fn create_procedure_instance( + &self, + operation: Option, + ) -> anyhow::Result { let program = self.program.clone(); let common = self.common.clone(); let load_balance_guard = self.load_balance_guard.clone(); @@ -320,15 +331,23 @@ impl JsModule { load_balance_guard, core_pinner, heap_policy, + self.execution_timeout, metrics, + operation, ) - .await - .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); - instance + .await?; + Ok(instance) } - pub async fn create_instance(&self) -> JsProcedureInstance { - self.create_procedure_instance().await + pub(in crate::host) async fn create_instance_for_operation( + &self, + operation: Option, + ) -> anyhow::Result { + self.create_procedure_instance(operation).await + } + + pub async fn create_instance(&self) -> anyhow::Result { + self.create_procedure_instance(None).await } } @@ -451,7 +470,8 @@ impl JsInstanceEnv { /// and friends. #[derive(Clone)] pub struct JsMainInstance { - tx: MeteredUnboundedSender, + tx: MeteredUnboundedSender>, + operation: Option, } /// A procedure instance for a [`JsModule`]. @@ -459,16 +479,48 @@ pub struct JsMainInstance { /// Procedure instances are checked out exclusively from the procedure pool and /// only execute procedure-style requests. pub struct JsProcedureInstance { - tx: mpsc::Sender, + tx: mpsc::Sender>, + operation: Option, + startup_failure: ProcedureStartupStatus, +} + +// Set only when a replacement generation fails before receiving another +// request. Publishing before closing the receiver proves queued calls did not +// execute. Unexplained worker exits and actual Rust panics remain fatal. +type ProcedureStartupStatus = Arc>; + +#[derive(Clone, Debug, thiserror::Error)] +#[error("procedure isolate startup failed: {0}")] +pub(in crate::host) struct JsProcedureStartupError(Arc); + +struct PhysicalRequest { + request: R, + operation: Option, } impl JsMainInstance { + pub(in crate::host) fn with_operation(mut self, operation: OperationLease) -> Self { + self.operation = Some(operation); + self + } + async fn request(&self, request: R) -> R::Response { - send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| request.into_worker_request(reply_tx)).await + send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| PhysicalRequest { + request: request.into_worker_request(reply_tx), + operation: self.operation.clone(), + }) + .await } async fn send_detached_request(&self, ctx: &'static str, request: JsMainWorkerRequest) { - if self.tx.send(request).is_err() { + if self + .tx + .send(PhysicalRequest { + request, + operation: self.operation.clone(), + }) + .is_err() + { panic!("JS worker exited before accepting `{ctx}`"); } } @@ -479,12 +531,14 @@ impl JsMainInstance { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, environment, + deployment, }) .await } @@ -544,8 +598,14 @@ impl JsMainInstance { &self, program: Program, environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { - self.request(InitDatabaseRequest { program, environment }).await + self.request(InitDatabaseRequest { + program, + environment, + deployment, + }) + .await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -630,6 +690,7 @@ js_main_request! { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, } => "update_database", anyhow::Result, UpdateDatabase } @@ -673,6 +734,7 @@ js_main_request! { InitDatabaseRequest { program: Program, environment: std::collections::BTreeMap, + deployment: Option, } => "init_database", anyhow::Result, InitDatabase } @@ -689,16 +751,24 @@ js_main_request! { } impl JsProcedureInstance { + pub(in crate::host) fn set_operation(&mut self, operation: Option) { + self.operation = operation; + } + pub(in crate::host) fn is_closed(&self) -> bool { - self.tx.is_closed() + self.startup_failure.get().is_some() || self.tx.is_closed() } async fn send_request( &self, ctx: &'static str, request: impl FnOnce(JsReplyTx) -> JsProcedureWorkerRequest, - ) -> T { - send_js_request(ctx, &self.tx, request).await + ) -> Result { + send_js_request(ctx, &self.tx, &self.startup_failure, |reply_tx| PhysicalRequest { + request: request(reply_tx), + operation: self.operation.clone(), + }) + .await } pub async fn call_procedure(&self, params: CallProcedureParams) -> CallProcedureReturn { @@ -707,6 +777,10 @@ impl JsProcedureInstance { params, }) .await + .unwrap_or_else(|error| CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(error.to_string())), + tx_offset: None, + }) } pub async fn call_http_handler( @@ -717,25 +791,38 @@ impl JsProcedureInstance { JsProcedureWorkerRequest::CallHttpHandler { reply_tx, params } }) .await + .map_err(|error| HttpHandlerCallError::InternalError(error.to_string()))? } - pub(in crate::host) async fn enqueue_procedure(&self, params: CallProcedureParams) -> JsProcedureCall { + pub(in crate::host) async fn enqueue_procedure( + &self, + params: CallProcedureParams, + ) -> Result { let (reply_tx, reply_rx) = oneshot::channel(); if self .tx - .send(JsProcedureWorkerRequest::CallProcedure { reply_tx, params }) + .send(PhysicalRequest { + request: JsProcedureWorkerRequest::CallProcedure { reply_tx, params }, + operation: self.operation.clone(), + }) .await .is_err() { + if let Some(error) = self.startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `call_procedure`"); } - JsProcedureCall { reply_rx } + Ok(JsProcedureCall { + reply_rx, + startup_failure: self.startup_failure.clone(), + }) } pub(in crate::host) async fn call_scheduled_procedure( &self, params: ScheduledFunctionParams, - ) -> CallScheduledFunctionResult { + ) -> Result { self.send_request("scheduled_procedure", |reply_tx| { JsProcedureWorkerRequest::ScheduledProcedure { reply_tx, params } }) @@ -746,26 +833,33 @@ impl JsProcedureInstance { async fn send_js_request( ctx: &'static str, tx: &mpsc::Sender, + startup_failure: &ProcedureStartupStatus, request: impl FnOnce(JsReplyTx) -> Req, -) -> T +) -> Result where Req: Send + 'static, { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).await.is_err() { + if let Some(error) = startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `{ctx}`"); } match reply_rx.await { - Ok(Ok(value)) => value, + Ok(Ok(value)) => Ok(value), Ok(Err(panic)) => panic::resume_unwind(panic), - Err(_) => panic!("JS worker exited before replying to `{ctx}`"), + Err(_) => match startup_failure.get() { + Some(error) => Err(error.clone()), + None => panic!("JS worker exited before replying to `{ctx}`"), + }, } } -async fn send_js_unbounded_request( +async fn send_js_unbounded_request( ctx: &'static str, - tx: &MeteredUnboundedSender, - request: impl FnOnce(JsReplyTx) -> JsMainWorkerRequest, + tx: &MeteredUnboundedSender, + request: impl FnOnce(JsReplyTx) -> Req, ) -> T { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).is_err() { @@ -785,11 +879,13 @@ pub(in crate::host) type JsFatalHook = Arc; pub(in crate::host) struct JsProcedureCall { reply_rx: oneshot::Receiver>, + startup_failure: ProcedureStartupStatus, } pub(in crate::host) enum JsProcedureCallCompletion { Completed(CallProcedureReturn), Panicked, + StartupFailed(JsProcedureStartupError), WorkerExited, } @@ -798,7 +894,10 @@ impl JsProcedureCall { match self.reply_rx.await { Ok(Ok(ret)) => JsProcedureCallCompletion::Completed(ret), Ok(Err(_panic)) => JsProcedureCallCompletion::Panicked, - Err(_) => JsProcedureCallCompletion::WorkerExited, + Err(_) => match self.startup_failure.get() { + Some(error) => JsProcedureCallCompletion::StartupFailed(error.clone()), + None => JsProcedureCallCompletion::WorkerExited, + }, } } } @@ -816,6 +915,7 @@ enum JsMainWorkerRequest { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -877,6 +977,7 @@ enum JsMainWorkerRequest { reply_tx: JsReplyTx>, program: Program, environment: std::collections::BTreeMap, + deployment: Option, }, } @@ -899,7 +1000,7 @@ enum JsProcedureWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 208); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T) { if reply_tx.send(Ok(value)).is_err() { @@ -1190,7 +1291,9 @@ fn startup_instance_worker<'scope>( scope: &mut PinScope<'scope, '_>, program: Arc, module_or_mcc: Either, + execution_timeout: Duration, ) -> anyhow::Result<(HookFunctions<'scope>, ModuleCommon)> { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; let hook_functions = catch_exception(scope, |scope| { // Start-up the user's module. let exports_obj = eval_user_module(scope, &program)?; @@ -1199,13 +1302,19 @@ fn startup_instance_worker<'scope>( let hooks = get_hooks(scope, exports_obj)?.ok_or_else(|| anyhow::anyhow!("must export schema as default export"))?; Ok(hooks) - })?; + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + let hook_functions = hook_functions?; // If we don't have a module, make one. let module_common = match module_or_mcc { Either::Left(module_common) => module_common, Either::Right(mcc) => { - let def = extract_description(scope, &hook_functions, &mcc.replica_ctx)?; + let def = extract_description(scope, &hook_functions, &mcc.replica_ctx, execution_timeout)?; // Validate and create a common module from the raw definition. build_common_module_from_raw(mcc, def)? @@ -1268,6 +1377,7 @@ async fn spawn_main_instance_worker( load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, JsMainInstance)> { spawn_instance_worker::( @@ -1276,18 +1386,23 @@ async fn spawn_main_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + None, ) .await } +#[allow(clippy::too_many_arguments)] async fn spawn_procedure_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1295,7 +1410,9 @@ async fn spawn_procedure_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + operation, ) .await } @@ -1314,9 +1431,9 @@ trait JsWorkerSpec { fn channel(database_identity: &Identity) -> (Self::Sender, Self::Receiver); - fn make_instance(tx: Self::Sender) -> Self::Instance; + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance; - fn blocking_recv(rx: &mut Self::Receiver) -> Option; + fn blocking_recv(rx: &mut Self::Receiver) -> Option>; fn handle_request( request: Self::Request, @@ -1330,8 +1447,8 @@ trait JsWorkerSpec { impl JsWorkerSpec for MainJsWorker { type Request = JsMainWorkerRequest; type Instance = JsMainInstance; - type Sender = MeteredUnboundedSender; - type Receiver = MeteredUnboundedReceiver; + type Sender = MeteredUnboundedSender>; + type Receiver = MeteredUnboundedReceiver>; const KIND: JsWorkerKind = JsWorkerKind::Main; @@ -1346,11 +1463,11 @@ impl JsWorkerSpec for MainJsWorker { ) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsMainInstance { tx } + fn make_instance(tx: Self::Sender, _startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsMainInstance { tx, operation: None } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1368,8 +1485,8 @@ impl JsWorkerSpec for MainJsWorker { impl JsWorkerSpec for ProcedureJsWorker { type Request = JsProcedureWorkerRequest; type Instance = JsProcedureInstance; - type Sender = mpsc::Sender; - type Receiver = mpsc::Receiver; + type Sender = mpsc::Sender>; + type Receiver = mpsc::Receiver>; const KIND: JsWorkerKind = JsWorkerKind::Procedure; @@ -1377,11 +1494,15 @@ impl JsWorkerSpec for ProcedureJsWorker { mpsc::channel(JS_PROCEDURE_INSTANCE_QUEUE_CAPACITY) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsProcedureInstance { tx } + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsProcedureInstance { + tx, + startup_failure, + operation: None, + } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1412,8 +1533,9 @@ fn handle_main_worker_request( old_module_info, policy, environment, + deployment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, environment, inst); + let res = instance_common.update_database(program, old_module_info, policy, environment, deployment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1514,10 +1636,18 @@ fn handle_main_worker_request( reply_tx, program, environment, + deployment, } => handle_worker_request("init_database", reply_tx, || { let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, bool) = - init_database(replica_ctx, &info.module_def, program, environment, call_reducer); + let (res, trapped): (Result, bool) = init_database( + replica_ctx, + &info.module_def, + info.module_hash, + program, + environment, + deployment, + call_reducer, + ); (res, trapped) }), } @@ -1585,21 +1715,24 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// Spawns an instance worker for `program` and returns on success the /// corresponding instance handle that talks to the worker. /// -/// When [`ModuleCommon`] is passed, it's assumed that this program has already -/// been validated. In that case, `Ok(_)` should be returned. +/// When [`ModuleCommon`] is passed, the program has already been validated. +/// Starting another isolate can still fail, including its execution deadline. /// /// Otherwise, when [`ModuleCreationContext`] is passed, this is the first time /// both the module and instance are created. /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. +#[allow(clippy::too_many_arguments)] async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, instance_metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1612,6 +1745,8 @@ where Either::Right(mcc) => mcc.replica_ctx.database_identity, }; let (request_tx, mut request_rx) = W::channel(&database_identity); + let startup_failure = ProcedureStartupStatus::default(); + let worker_startup_failure = startup_failure.clone(); let rt = tokio::runtime::Handle::current(); @@ -1624,6 +1759,7 @@ where let mut startup_result_tx = Some(result_tx); let mut module_common_for_recreate = None::; + let mut physical_operation = operation; 'worker: loop { let replacing_instance = module_common_for_recreate.is_some(); let generation_start_time = replacing_instance.then(Instant::now); @@ -1656,7 +1792,7 @@ where .expect("our builtin code shouldn't error"); // Setup the JS module, find call_reducer, and maybe build the module. - startup_instance_worker(scope, program.clone(), generation_module_or_mcc) + startup_instance_worker(scope, program.clone(), generation_module_or_mcc, execution_timeout) })); let (hooks, module_common) = match startup_result { @@ -1679,6 +1815,7 @@ where log::warn!("startup result receiver disconnected"); } } else { + let _ = worker_startup_failure.set(JsProcedureStartupError(format!("{err:#}").into())); log::error!("failed to restart JS worker: {err:#}"); } return; @@ -1730,6 +1867,7 @@ where .v8_heap_limit_hit .with_label_values(&info.database_identity), initial_heap_limit: heap_policy.heap_limit_bytes, + execution_timeout, }; let _initial_heap_stats = record_heap_stats(inst.scope, &mut heap_metrics); @@ -1737,9 +1875,11 @@ where // // The loop is terminated when the last worker instance handle is dropped. // This will cause channels, scopes, and the isolate to be cleaned up. + physical_operation.take(); let mut requests_since_heap_check = 0u64; let mut last_heap_check_at = Instant::now(); - while let Some(request) = W::blocking_recv(&mut request_rx) { + while let Some(PhysicalRequest { request, operation }) = W::blocking_recv(&mut request_rx) { + physical_operation = operation; core_pinner.pin_if_changed(); let mut outcome = @@ -1770,7 +1910,9 @@ where } match outcome { - WorkerRequestOutcome::Continue => {} + WorkerRequestOutcome::Continue => { + physical_operation.take(); + } WorkerRequestOutcome::RecreateInstance => { instance_metrics.track_instance_removed(); continue 'worker; @@ -1786,7 +1928,7 @@ where // Get the module, if any, and get any setup errors from the worker. let res: Result = result_rx.await.expect("should have a sender"); res.map(|opt_mc| { - let inst = W::make_instance(request_tx); + let inst = W::make_instance(request_tx, startup_failure); (opt_mc, inst) }) } @@ -1876,11 +2018,12 @@ struct V8Instance<'a, 'scope, 'isolate> { /// Metric for the number of times the v8 heap limit has been hit. heap_limit_hit_metric: &'a IntCounter, initial_heap_limit: usize, + execution_timeout: Duration, } impl WasmInstance for V8Instance<'_, '_, '_> { fn extract_descriptions(&mut self) -> Result { - extract_description(self.scope, self.hooks, self.replica_ctx) + extract_description(self.scope, self.hooks, self.replica_ctx, self.execution_timeout) } fn replica_ctx(&self) -> &Arc { @@ -2001,13 +2144,14 @@ where // are released when the reducer/view/procedure returns. v8::scope!(let scope, scope); - // TODO(v8): Start the budget timeout and long-running logger. let env = env_on_isolate_unwrap(scope); // Start the timer. // We'd like this tightly around `call`. env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); env.instance_env.set_call_auth_flags(op.call_auth_flags()); + env.instance_env + .set_hosted_auth(op.hosted_auth().map(|proof| std::sync::Arc::new(proof.clone()))); // Wrap the call in `TryCatch`. // @@ -2016,7 +2160,12 @@ where // opened by the caller before entering `common_call`. v8::tc_scope!(let scope, scope); - let call_result = call(scope, inst.hooks, op).map_err(|mut e| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), inst.execution_timeout); + let mut call_result = match &deadline { + Ok(_) => call(scope, inst.hooks, op), + Err(error) => Err(anyhow::anyhow!("cannot start JavaScript execution deadline: {error}").into()), + } + .map_err(|mut e| { if let ErrorOrException::Exception(_) = e { // If we're terminating execution, don't try to check `instanceof`. if scope.can_continue() @@ -2036,10 +2185,7 @@ where // We can continue. ExecutionError::Recoverable(e.unwrap_or_else(Into::into)) } else if scope.has_terminated() { - // We can continue if we do `Isolate::cancel_terminate_execution`. - // Must be called *after* we check `has_terminated()`, or else it will - // cause it to return `false`. - scope.cancel_terminate_execution(); + // Reset only after synchronizing with this invocation's timer. let e = e.unwrap_or_else(|unknown| termination_error.unwrap_or_else(|| unknown.into())); ExecutionError::Recoverable(e) } else { @@ -2048,6 +2194,14 @@ where } }); + // The timer also covers exception inspection, which can execute user + // getters or Symbol.hasInstance. Synchronizing with it before resetting prevents a + // delayed timer from terminating the next invocation of this isolate. + if deadline.is_ok_and(ExecutionDeadline::finish) { + // Even a call that returned at the exact boundary must roll back. + call_result = Err(ExecutionError::Recoverable(ExecutionTimedOut.into())); + } + // Ensure there's no lingering termination request. termination_flag.clear(); scope.cancel_terminate_execution(); @@ -2084,14 +2238,22 @@ fn extract_description<'scope>( scope: &mut PinScope<'scope, '_>, hooks: &HookFunctions<'_>, replica_ctx: &ReplicaContext, + execution_timeout: Duration, ) -> Result { run_describer( |a, b, c| log_traceback(replica_ctx, a, b, c), || { - Ok(catch_exception(scope, |scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; + let result = catch_exception(scope, |scope| { let def = call_describe_module(scope, hooks)?; Ok(def) - })?) + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + Ok(result?) }, ) } @@ -2134,6 +2296,7 @@ mod test { caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, call_auth_flags: 0, + hosted_auth: None, timestamp: Timestamp::from_micros_since_unix_epoch(24), args: &ArgsTuple::nullary(), }; diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 03421af0437..14030b381b0 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -45,6 +45,7 @@ pub fn call_call_procedure( caller_identity: sender, caller_connection_id: connection_id, call_auth_flags: _, + hosted_auth: _, timestamp, arg_bytes: procedure_args, } = op; @@ -450,13 +451,19 @@ pub fn console_log<'scope>( let mut buf = scratch_buf::<128>(); let msg = msg.to_rust_cow_lossy(scope, &mut buf); - let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) - .ok_or_else(exception_already_thrown)? - .get_frame(scope, 1) - .ok_or_else(exception_already_thrown)?; + let trace = v8::StackTrace::current_stack_trace(scope, 2).ok_or_else(exception_already_thrown)?; + // The normal bindings add a logging wrapper, but modules may call this + // syscall directly, including from top-level code. V8's GetFrame does not + // check the index in release builds, so never request an absent frame. + let frame = match trace.get_frame_count() { + 0 => None, + 1 => trace.get_frame(scope, 0), + _ => trace.get_frame(scope, 1), + }; + let line_number = frame.map(|frame| frame.get_line_number() as u32); let mut buf = scratch_buf::<32>(); let filename = frame - .get_script_name(scope) + .and_then(|frame| frame.get_script_name(scope)) .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); let level = (level as u8).into(); @@ -470,7 +477,7 @@ pub fn console_log<'scope>( tracing::error!( "`JsInstanceEnv` unavailable while processing guest log at {}:{}: {msg}", filename.as_deref().unwrap_or("unknown"), - frame.get_line_number() + line_number.unwrap_or_default() ); })?; @@ -480,7 +487,7 @@ pub fn console_log<'scope>( ts: InstanceEnv::now_for_logging(), target: None, filename: filename.as_deref(), - line_number: Some(frame.get_line_number() as u32), + line_number, function, message: &msg, }; diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index 4d6783465df..ab852eda401 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -496,6 +496,7 @@ pub(super) fn call_call_reducer( caller_identity: sender, caller_connection_id: conn_id, call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index bb441a2a89c..54473e67d3c 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -480,6 +480,7 @@ pub(super) fn call_call_reducer<'scope>( caller_identity: sender, caller_connection_id: conn_id, call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 159814e44a9..32bdf0bc68a 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1,7 +1,10 @@ use super::instrumentation::CallTimes; use super::*; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::client::ClientActorId; use crate::database_logger; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit}; use crate::db::sql::ast::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; @@ -357,7 +360,7 @@ pub enum InitializationError { #[error(transparent)] Validation(#[from] ValidationError), #[error(transparent)] - ModuleValidation(#[from] spacetimedb_schema::error::ValidationErrors), + ModuleValidation(#[from] Box), #[error("setup function returned an error: {0}")] Setup(Box), #[error("wasm trap while calling {func:?}")] @@ -500,9 +503,16 @@ impl WasmModuleInstance { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { - self.common - .update_database(program, old_module_info, policy, environment, &mut self.instance) + self.common.update_database( + program, + old_module_info, + policy, + environment, + deployment, + &mut self.instance, + ) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -560,11 +570,21 @@ impl WasmModuleInstance { &mut self, program: Program, environment: std::collections::BTreeMap, + deployment: Option, ) -> anyhow::Result { - let module_def = &self.common.info.clone().module_def; + let info = self.common.info.clone(); + let module_def = &info.module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx_offset(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, environment, call_reducer); + let (res, trapped) = init_database( + replica_ctx, + module_def, + info.module_hash, + program, + environment, + deployment, + call_reducer, + ); self.trapped = trapped; res } @@ -701,12 +721,51 @@ impl InstanceCommon { old_module_info: Arc, policy: MigrationPolicy, environment: std::collections::BTreeMap, + deployment: Option, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); let stdb = &replica_ctx.relational_db(); + let timestamp = Timestamp::now(); + let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, admission) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<_> { + ensure!( + self.info.module_hash == program.hash, + "program does not match the instantiated module" + ); + let admission = if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, &self.info.module_def)?; + deployment::check_deployment_commit(tx, request, timestamp, &Default::default())? + } else { + deployment::require_unmanaged_publication(tx)?; + CommitAdmission::Ready + }; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + return Ok(admission); + } + deployment::validate_active_hosted_grants(tx, &self.info.module_def)?; + use spacetimedb_datastore::system_tables::{read_hash_from_col, StModuleFields, ST_MODULE_ID}; + let row = tx.iter(ST_MODULE_ID)?.next().context("database is not initialized")?; + let stored_hash = read_hash_from_col(row, StModuleFields::ProgramHash)?; + ensure!( + stored_hash == old_module_info.module_hash, + "module changed before publication admission" + ); + Ok(admission) + })?; + if let CommitAdmission::AlreadyCommitted(result) = admission { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + let (sender, tx_offset) = tokio::sync::oneshot::channel(); + let _ = sender.send(offset); + return Ok(UpdateDatabaseResult::DeploymentAlreadyCommitted { + result, + tx_offset, + durable_offset: stdb.durable_tx_offset(), + }); + } let plan: MigratePlan = match policy.try_migrate( self.info.database_identity, old_module_info.module_hash, @@ -716,30 +775,23 @@ impl InstanceCommon { ) { Ok(plan) => plan, Err(e) => { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); return match e { MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())), _ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())), - } + }; } }; let program_hash = program.hash; let host_type = HostType::from(program.kind); - let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { - use spacetimedb_datastore::system_tables::{StModuleFields, ST_MODULE_ID}; - let row = tx - .iter(ST_MODULE_ID)? - .next() - .context("database program is not initialized")?; - let current_hash = - spacetimedb_datastore::system_tables::read_hash_from_col(row, StModuleFields::ProgramHash)?; - anyhow::ensure!( - current_hash == old_module_info.module_hash, - "database program changed before publication" - ); crate::db::environment::replace(stdb, tx, self.info.module_def.environment(), &environment)?; stdb.update_program(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } Ok(()) })?; system_logger.info(&format!("Updated program to {program_hash}")); @@ -822,7 +874,9 @@ impl InstanceCommon { tx: MutTxId, inst: &mut I, ) -> Result<(ViewCallResult, u32, bool), anyhow::Error> { - let view_calls = collect_subscribed_view_calls(&tx, &self.info.module_def, self.info.owner_identity)?; + let (tx, view_calls) = inst.replica_ctx().relational_db().with_auto_rollback(tx, |tx| { + collect_subscribed_view_calls(tx, &self.info.module_def, self.info.owner_identity) + })?; Ok(self.execute_view_calls(tx, view_calls, inst)) } @@ -839,10 +893,27 @@ impl InstanceCommon { let CallProcedureParams { timestamp, caller_identity, + hosted_auth, timer, .. } = params; + let admission = inst + .replica_ctx() + .relational_db() + .with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, inst.replica_ctx().relational_db(), hosted_auth.as_deref()) + }); + if let Err(err) = admission { + return ( + CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(err.to_string())), + tx_offset: None, + }, + false, + ); + } + // TODO(observability): Add tracing spans, energy, metrics? // These will require further thinking once we implement procedure suspend/resume, // and so are not worth doing yet. @@ -1009,6 +1080,7 @@ impl InstanceCommon { caller_identity, caller_connection_id, call_auth_flags, + hosted_auth, client, request_id, reducer_id, @@ -1033,12 +1105,44 @@ impl InstanceCommon { caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, call_auth_flags, + hosted_auth, timestamp, args: &args, }; let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); + if let Err(err) = check_hosted_admission(&tx, stdb, op.hosted_auth.as_deref()) { + let event = ModuleEvent { + timestamp, + caller_identity, + caller_connection_id: caller_connection_id_opt, + function_call: ModuleFunctionCall { + reducer: Some(reducer_name.clone()), + reducer_id, + args, + }, + status: EventStatus::FailedInternal(err.to_string()), + reducer_return_value: None, + execution_budget_used: FunctionBudget::ZERO, + host_execution_duration: Default::default(), + request_id, + timer, + }; + let CommitAndBroadcastEventSuccess { event, tx_offset, .. } = + commit_and_broadcast_event(&info.subscriptions, client, event, tx); + return ( + ReducerCallResultWithTxOffset { + result: ReducerCallResult { + outcome: ReducerOutcome::from(&event.status), + execution_budget_used: FunctionBudget::ZERO, + execution_duration: Default::default(), + }, + tx_offset, + }, + false, + ); + } let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -1901,6 +2005,9 @@ pub trait InstanceOp { fn call_auth_flags(&self) -> u32 { 0 } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + None + } } /// Describes a view call in a cheaply shareable way. @@ -1962,12 +2069,19 @@ pub struct ReducerOp<'a> { pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, } impl InstanceOp for ReducerOp<'_> { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &NamespacedIdentifier { self.name.as_namespaced() } @@ -1977,9 +2091,6 @@ impl InstanceOp for ReducerOp<'_> { fn call_type(&self) -> FuncCallType { FuncCallType::Reducer } - fn call_auth_flags(&self) -> u32 { - self.call_auth_flags - } } impl From> for execution_context::ReducerContext { @@ -1990,6 +2101,7 @@ impl From> for execution_context::ReducerContext { caller_identity, caller_connection_id, call_auth_flags: _, + hosted_auth: _, timestamp, args, }: ReducerOp<'_>, @@ -2012,6 +2124,7 @@ pub struct ProcedureOp { pub caller_identity: Identity, pub caller_connection_id: ConnectionId, pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, pub arg_bytes: Bytes, } @@ -2029,6 +2142,7 @@ impl ProcedureOp { caller_identity: params.caller_identity, caller_connection_id: params.caller_connection_id, call_auth_flags: params.call_auth_flags, + hosted_auth: params.hosted_auth.clone(), timestamp: params.timestamp, arg_bytes: params.args.get_bsatn().clone(), }, @@ -2039,6 +2153,12 @@ impl ProcedureOp { } impl InstanceOp for ProcedureOp { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &NamespacedIdentifier { &self.name } @@ -2048,9 +2168,6 @@ impl InstanceOp for ProcedureOp { fn call_type(&self) -> FuncCallType { FuncCallType::Procedure } - fn call_auth_flags(&self) -> u32 { - self.call_auth_flags - } } /// Describes an HTTP handler call in a cheaply shareable way. diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 9770a311407..114ed910086 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -340,14 +340,6 @@ impl WasmInstanceEnv { self.bytes_sinks.remove(&sink).unwrap_or_default() } - pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { - caller.data().instance_env.get_call_auth_flags() - } - - pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { - self.instance_env.set_call_auth_flags(flags); - } - /// Signal to this `WasmInstanceEnv` that a reducer or procedure call is beginning. /// /// Returns the handle used by reducers and procedures to read from `args` @@ -1594,34 +1586,6 @@ impl WasmInstanceEnv { }) } - /// Read an environment value as a nullable BytesSource. Zero means missing; - /// a present empty string always receives a nonzero, consumable source. - pub fn env_get( - caller: Caller<'_, Self>, - key: WasmPtr, - key_len: u32, - target_ptr: WasmPtr, - ) -> RtResult { - Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { - if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { - return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); - } - let (mem, env) = Self::mem_env(caller); - let key = mem.deref_str(key, key_len)?; - match env.instance_env.env_get(key)? { - None => Ok(0), - Some(value) => { - // These buffers live on the host heap until consumed or the - // invocation ends. Bound retained reads from hand-written Wasm. - if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { - return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); - } - Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) - } - } - }) - } - /// Finds the JWT payload associated with `connection_id`. /// A `[ByteSourceId]` for the payload will be written to `target_ptr`. /// If nothing is found for the connection, `[ByteSourceId::INVALID]` (zero) is written to `target_ptr`. @@ -1664,6 +1628,51 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + + /// Returns host-verified invocation flags. Bit 0 is internal authority. + /// This does not read tables and is available outside transactions. + pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { + caller.data().instance_env.get_call_auth_flags() + } + + pub(crate) fn set_hosted_auth( + &mut self, + auth: Option>, + ) { + self.instance_env.set_hosted_auth(auth); + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.instance_env.set_call_auth_flags(flags); + } + /// Writes the identity of the module into `out = out_ptr[..32]`. /// /// # Traps diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 13c71b89343..163be82466e 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -648,6 +648,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { .data_mut() .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let call_result = call_sync_typed_func( &self.call_reducer, @@ -772,6 +773,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { .data_mut() .start_funcall(op.name().clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 1f2c587b762..ffd99c9b991 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,3 +1,4 @@ +use crate::auth::invocation::{check_hosted_admission, SqlCallAuth}; use std::sync::Arc; use std::time::Duration; @@ -10,7 +11,7 @@ use crate::host::module_host::{ WasmInstance, }; use crate::host::{ArgsTuple, ModuleHost}; -use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::TransactionOffset; use crate::subscription::tx::DeltaTx; use anyhow::anyhow; @@ -18,6 +19,7 @@ use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_engine::relational_db::RelationalDB; use spacetimedb_expr::statement::Statement; +#[cfg(test)] use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::Timestamp; @@ -52,11 +54,15 @@ pub struct SqlResult { pub async fn run( db: Arc, sql_text: String, - auth: AuthCtx, + auth: impl Into, subs: Option, module: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + let auth = auth.into(); + if auth.hosted.is_some() && module.is_none() { + return Err(anyhow!("hosted SQL requires a module with hosted authentication capability").into()); + } match module { Some(module) => module.call_sql(db, sql_text, auth, subs, head).await, None => run_inner::(None, db, sql_text, auth, subs, head).map(|x| x.0), @@ -70,7 +76,7 @@ pub(crate) fn run_with_instance( instance: &mut RefInstance, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { @@ -81,13 +87,14 @@ fn run_inner( instance: Option<&mut RefInstance>, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_deref())?; let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; // Check mutation authority while the automatic rollback guard owns // the transaction, including rejected administrative statements. @@ -104,6 +111,13 @@ fn run_inner( "Database environment variables can only be changed by publishing" )); } + if let Statement::DML(dml) = &stmt + && spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) + { + return Err(anyhow!( + "Deployment and container authorization metadata may only be changed by the host" + )); + } Ok(stmt) })?; @@ -117,7 +131,7 @@ fn run_inner( None => (tx, false), }; - let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql); + let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql)?; let (tx_offset_send, tx_offset) = oneshot::channel(); // Release the tx on drop, so that we record metrics @@ -222,7 +236,10 @@ fn run_inner( request_id: None, timer: None, }; - let res = commit_and_broadcast_event(&subs.unwrap(), None, event, tx); + let res = subs + .unwrap() + .commit_and_broadcast_event(None, event, tx)? + .map_err(|_| anyhow!("SQL transaction write conflict"))?; Ok(( SqlResult { tx_offset: res.tx_offset, @@ -457,6 +474,25 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn owner_sql_cannot_rewrite_container_authority_or_deployment() -> ResultTest<()> { + let db = TestDB::in_memory()?; + for table in [ + "st_deployment", + "st_publish_fence", + "st_deployment_operation", + "st_container_fence", + "st_connection_auth", + ] { + // This uses the owner test context. Permission to mutate application + // tables does not grant permission to replace host operational state. + let error = run_for_testing(&db, &format!("DELETE FROM {table}")).unwrap_err(); + assert!(error.to_string().contains("may only be changed by the host"), "{error}"); + assert!(run_for_testing(&db, &format!("SELECT * FROM {table}"))?.is_empty()); + } + Ok(()) + } + #[test] fn test_limit() -> ResultTest<()> { let (db, _) = create_data(5)?; diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index c88da72e8d2..11a2913f0ce 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -7,6 +7,7 @@ use super::module_subscription_manager::{ use super::query::{compile_query_with_hashes, CompiledQuery}; use super::tx::DeltaTx; use super::TableUpdateType; +use crate::auth::invocation::check_hosted_admission; use crate::client::messages::{ ProcedureResultMessage, SerializableMessage, SubscriptionData, SubscriptionError, SubscriptionMessage, SubscriptionResult, SubscriptionRows, SubscriptionUpdateMessage, TransactionUpdateMessage, @@ -31,6 +32,7 @@ use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet} use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, TxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, TxData}; use spacetimedb_durability::TxOffset; @@ -66,6 +68,7 @@ pub struct ModuleSubscriptions { stats: Arc, metrics: Arc, module_def_version: Arc, + hosted_connections: Arc>>>, } #[derive(Debug, Clone)] @@ -341,6 +344,7 @@ impl ModuleSubscriptions { Self { relational_db, subscriptions, + hosted_connections: Arc::new(RwLock::new(Vec::new())), broadcast_queue, stats, metrics, @@ -351,6 +355,46 @@ impl ModuleSubscriptions { } } + /// Register every hosted socket, even if it has no subscriptions. Taking the + /// database transaction before the registry lock serializes with fencing. + pub(crate) fn register_hosted_connection(&self, sender: &Arc) -> anyhow::Result<()> { + self.relational_db.with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, &self.relational_db, sender.auth.hosted.as_ref())?; + let mut connections = self.hosted_connections.write(); + connections.retain(|connection| connection.strong_count() != 0); + connections.push(Arc::downgrade(sender)); + Ok(()) + }) + } + + pub(crate) fn unregister_hosted_connection(&self, sender: &std::sync::Weak) { + self.hosted_connections + .write() + .retain(|connection| connection.strong_count() != 0 && !connection.ptr_eq(sender)); + } + + #[cfg(test)] + pub(crate) fn hosted_connection_count(&self) -> usize { + self.hosted_connections.read().len() + } + + /// Call with the transaction containing the new target fence. After its + /// durable commit, await every returned actor's completion before acking the + /// fence barrier. This also cancels socket batches already dequeued for I/O. + pub fn cancel_invalid_hosted_connections(&self, tx: &S) -> Vec { + let mut cancelled = Vec::new(); + self.hosted_connections.write().retain(|connection| { + let Some(connection) = connection.upgrade() else { + return false; + }; + if check_hosted_admission(tx, &self.relational_db, connection.auth.hosted.as_ref()).is_err() { + cancelled.push(connection.cancel_hosted_connection()); + } + true + }); + cancelled + } + pub fn set_module_def_version(&self, version: RawModuleDefVersion) { self.module_def_version .store(Self::encode_module_def_version(version), Ordering::Release); @@ -370,7 +414,8 @@ impl ModuleSubscriptions { fn decode_module_def_version(version: u8) -> RawModuleDefVersion { match version { 1 => RawModuleDefVersion::V10, - _ => RawModuleDefVersion::V9OrEarlier, + 0 => RawModuleDefVersion::V9OrEarlier, + _ => unreachable!("invalid stored module definition version"), } } @@ -685,6 +730,7 @@ impl ModuleSubscriptions { let hash_with_param = QueryHash::from_string(&sql, auth.caller(), true); let (mut_tx, _) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let existing_query = { let guard = self.subscriptions.read(); @@ -794,6 +840,8 @@ impl ModuleSubscriptions { ) }; + let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let queries = { let mut subscriptions = self.subscriptions.write(); return_on_err!( @@ -812,7 +860,8 @@ impl ModuleSubscriptions { return Ok(None); }; - let (mut tx, tx_offset) = self.unsubscribe_views(query, auth.caller())?; + let mut_tx = ScopeGuard::::into_inner(mut_tx); + let (mut tx, tx_offset) = self.unsubscribe_views_and_downgrade_tx(mut_tx, query, auth.caller())?; let (table_rows, metrics) = return_on_err_with_sql!( self.evaluate_initial_subscription(sender.clone(), query.clone(), &tx, TableUpdateType::Unsubscribe), @@ -875,6 +924,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -990,6 +1040,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1063,7 +1114,7 @@ impl ModuleSubscriptions { /// If either one is currently tracked, we can avoid recompilation. fn compile_queries( &self, - sender: Identity, + sender: &ClientConnectionSender, auth: AuthCtx, queries: &[Box], num_queries: usize, @@ -1079,13 +1130,14 @@ impl ModuleSubscriptions { subscribe_to_all_tables = true; continue; } - let hash = QueryHash::from_string(sql, sender, false); - let hash_with_param = QueryHash::from_string(sql, sender, true); + let hash = QueryHash::from_string(sql, sender.id.identity, false); + let hash_with_param = QueryHash::from_string(sql, sender.id.identity, true); query_hashes.push((sql, hash, hash_with_param)); } // We always get the db lock before the subscription lock to avoid deadlocks. let (mut_tx, _tx_offset) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let compile_timer = metrics.compilation_time.start_timer(); @@ -1355,13 +1407,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer: _compile_timer, } = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1463,13 +1509,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer, } = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1627,7 +1667,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer, } = self.compile_queries( - sender.id.identity, + &sender, auth, &subscription.query_strings, num_queries, @@ -1757,7 +1797,7 @@ impl ModuleSubscriptions { // We'll later ensure tx is released/cleaned up once out of scope. let (read_tx, tx_data, tx_metrics_mut) = match &mut event.status { EventStatus::Committed(db_update) => { - let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update); + let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update)?; *db_update = DatabaseUpdate::from_writes(&tx_data); (read_tx, tx_data, tx_metrics) } @@ -1865,7 +1905,7 @@ impl ModuleSubscriptions { sender: Identity, ) -> Result<(TxGuard, TransactionOffset), DBError> { Self::_unsubscribe_views(&mut tx, view_collector, sender)?; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); Ok(self.guard_tx(tx, opts)) } @@ -1906,7 +1946,7 @@ impl ModuleSubscriptions { (tx, trapped) = ModuleHost::materialize_views(tx, instance, view_collector, sender, Workload::Subscribe)?; }; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); let (a, b) = self.guard_tx(tx, opts); diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index d21706074e1..6509fba550c 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -1775,7 +1775,13 @@ pub struct BroadcastQueue(SenderWithGauge); #[derive(thiserror::Error, Debug)] #[error(transparent)] -pub struct BroadcastError(#[from] mpsc::error::SendError); +pub struct BroadcastError(Box>); + +impl From> for BroadcastError { + fn from(error: mpsc::error::SendError) -> Self { + Self(Box::new(error)) + } +} impl BroadcastQueue { fn send(&self, message: SendWorkerMessage) -> Result<(), BroadcastError> { diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 3f61903e645..20537f7a0ff 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -358,6 +358,9 @@ impl CommittedState { self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); self.create_table(ST_ENV_ID, schemas[ST_ENV_IDX].clone()); + for schema in crate::system_tables::deployment_system_schemas() { + self.create_table(schema.table_id, schema.into()); + } // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 1082e0ae9c6..8cfa937c416 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1092,7 +1092,10 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; - use crate::system_tables::{ST_ENV_ID, ST_ENV_NAME}; + use crate::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_ENV_NAME, ST_PUBLISH_FENCE_ID, + }; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1560,6 +1563,12 @@ pub(crate) mod tests { TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_ENV_ID.into(), name: ST_ENV_NAME, ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_ID.into(), name: "st_deployment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_PUBLISH_FENCE_ID.into(), name: "st_publish_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_OPERATION_ID.into(), name: "st_deployment_operation", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_FENCE_ID.into(), name: "st_container_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONNECTION_AUTH_ID.into(), name: "st_connection_auth", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_ENVIRONMENT_ID.into(), name: "st_container_environment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1659,6 +1668,28 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 1, name: "revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 2, name: "last_operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 3, name: "payload", ty: AlgebraicType::bytes() }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 1, name: "publication_epoch", ty: AlgebraicType::U64 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 2, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 0, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 1, name: "previous_revision", ty: AlgebraicType::option(AlgebraicType::U256) }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 2, name: "committed_revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 3, name: "commit_result", ty: AlgebraicType::bytes() }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 4, name: "expires_at", ty: AlgebraicType::I64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 0, name: "source_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 1, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 2, name: "target_grant_revision", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 3, name: "target_set_hash", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 4, name: "allowed", ty: AlgebraicType::Bool }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 0, name: "connection_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 1, name: "sender_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 2, name: "call_auth_flags", ty: AlgebraicType::U32 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 0, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 1, name: "payload", ty: AlgebraicType::bytes() }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1692,6 +1723,12 @@ pub(crate) mod tests { IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col_list![0], name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col_list![0], name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col_list![0], name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col_list![0], name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col_list![0], name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col_list![0], name: "st_container_environment_generation_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1738,6 +1775,12 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col_list![0], constraint_name: "st_env_key_key", }, + ConstraintRow { constraint_id: 27, table_id: ST_DEPLOYMENT_ID.into(), unique_columns: col_list![0], constraint_name: "st_deployment_key_key", }, + ConstraintRow { constraint_id: 28, table_id: ST_PUBLISH_FENCE_ID.into(), unique_columns: col_list![0], constraint_name: "st_publish_fence_key_key", }, + ConstraintRow { constraint_id: 29, table_id: ST_DEPLOYMENT_OPERATION_ID.into(), unique_columns: col_list![0], constraint_name: "st_deployment_operation_operation_id_key", }, + ConstraintRow { constraint_id: 30, table_id: ST_CONTAINER_FENCE_ID.into(), unique_columns: col_list![0], constraint_name: "st_container_fence_source_identity_key", }, + ConstraintRow { constraint_id: 31, table_id: ST_CONNECTION_AUTH_ID.into(), unique_columns: col_list![0], constraint_name: "st_connection_auth_connection_id_key", }, + ConstraintRow { constraint_id: 32, table_id: ST_CONTAINER_ENVIRONMENT_ID.into(), unique_columns: col_list![0], constraint_name: "st_container_environment_generation_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2172,6 +2215,12 @@ pub(crate) mod tests { IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col_list![0], name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col_list![0], name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col_list![0], name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col_list![0], name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col_list![0], name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col_list![0], name: "st_container_environment_generation_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 042fa6cfc5f..56fc9a112ff 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -11,9 +11,10 @@ use super::{ use crate::{ error::ViewError, system_tables::{ - system_tables, ConnectionIdViaU128, StConnectionCredentialsFields, StConnectionCredentialsRow, - StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, StViewSubFields, - ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, + system_tables, ConnectionIdViaU128, StConnectionAuthFields, StConnectionCredentialsFields, + StConnectionCredentialsRow, StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, + StViewSubFields, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, + ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, }, }; use crate::{ @@ -3271,7 +3272,13 @@ impl MutTxId { ); } } - self.delete_st_client_credentials(database_identity, connection_id) + self.delete_st_client_credentials(database_identity, connection_id)?; + self.delete_col_eq( + ST_CONNECTION_AUTH_ID, + StConnectionAuthFields::ConnectionId.col_id(), + &ConnectionIdViaU128::from(connection_id).into(), + )?; + Ok(()) } /// Look up a client row by identity and connection ID in the `st_clients` system table. diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 61dcd11be9b..c8c1cb56ae9 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -175,7 +175,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_CONNECTION_CREDENTIALS_ID | ST_CONNECTION_AUTH_ID | ST_CONTAINER_ENVIRONMENT_ID => false, // We don't define any system views, so none of the view-related tables can be system meta-descriptors. ST_VIEW_ID | ST_VIEW_PARAM_ID | ST_VIEW_COLUMN_ID | ST_VIEW_SUB_ID | ST_VIEW_ARG_ID => false, ST_EVENT_TABLE_ID => { @@ -207,7 +207,9 @@ pub enum SystemTable { st_event_table = ST_EVENT_TABLE_ID.0 as _, } -pub fn system_tables() -> [TableSchema; 21] { +pub fn system_tables() -> [TableSchema; 27] { + let [deployment, publish_fence, deployment_operation, container_fence, connection_auth, container_environment] = + deployment_system_schemas(); [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -231,6 +233,12 @@ pub fn system_tables() -> [TableSchema; 21] { st_index_accessor_schema(), st_column_accessor_schema(), st_env_schema(), + deployment, + publish_fence, + deployment_operation, + container_fence, + connection_auth, + container_environment, ] } @@ -317,6 +325,8 @@ macro_rules! st_fields_enum { mod environment; pub use environment::*; +mod deployment; +pub use deployment::*; // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { @@ -676,6 +686,7 @@ fn system_module_def() -> ModuleDef { .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); environment::register_table(&mut builder); + deployment::register_tables(&mut builder); let result = builder .finish() @@ -703,6 +714,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); validate_system_table::(&result, ST_ENV_NAME); + deployment::validate_tables(&result); result } @@ -752,6 +764,7 @@ lazy_static::lazy_static! { m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); m.insert("st_env_key_key", ConstraintId(26)); + m.extend(deployment::CONSTRAINTS); m }; } @@ -791,6 +804,7 @@ lazy_static::lazy_static! { m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); m.insert("st_env_key_idx_btree", IndexId(30)); + m.extend(deployment::INDEXES); m }; } @@ -981,7 +995,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), ST_ENV_ID => Some(st_env_schema()), - _ => None, + table => deployment::system_schema(table), } } diff --git a/crates/datastore/src/system_tables/deployment.rs b/crates/datastore/src/system_tables/deployment.rs new file mode 100644 index 00000000000..4a793b6ac8a --- /dev/null +++ b/crates/datastore/src/system_tables/deployment.rs @@ -0,0 +1,253 @@ +//! Stable system schemas for container deployment state. +//! +//! The deployment payload and operation result have versioned binary encodings, +//! so adding a protocol version does not change the system table row layout. +//! Publication and target fences are operational metadata: application restore +//! must preserve/reconcile their current authority before admitting execution. + +use super::*; + +pub const ST_DEPLOYMENT_ID: TableId = TableId(22); +pub const ST_PUBLISH_FENCE_ID: TableId = TableId(23); +pub const ST_DEPLOYMENT_OPERATION_ID: TableId = TableId(24); +pub const ST_CONTAINER_FENCE_ID: TableId = TableId(25); +pub const ST_CONNECTION_AUTH_ID: TableId = TableId(26); +pub const ST_CONTAINER_ENVIRONMENT_ID: TableId = TableId(27); + +pub const ST_DEPLOYMENT_NAME: &str = "st_deployment"; +pub const ST_PUBLISH_FENCE_NAME: &str = "st_publish_fence"; +pub const ST_DEPLOYMENT_OPERATION_NAME: &str = "st_deployment_operation"; +pub const ST_CONTAINER_FENCE_NAME: &str = "st_container_fence"; +pub const ST_CONNECTION_AUTH_NAME: &str = "st_connection_auth"; +pub const ST_CONTAINER_ENVIRONMENT_NAME: &str = "st_container_environment"; + +st_fields_enum!(enum StDeploymentFields { + "key", Key = 0, + "revision", Revision = 1, + "last_operation_id", LastOperationId = 2, + "payload", Payload = 3, +}); +st_fields_enum!(enum StPublishFenceFields { + "key", Key = 0, + "publication_epoch", PublicationEpoch = 1, + "operation_id", OperationId = 2, +}); +st_fields_enum!(enum StDeploymentOperationFields { + "operation_id", OperationId = 0, + "previous_revision", PreviousRevision = 1, + "committed_revision", CommittedRevision = 2, + "commit_result", CommitResult = 3, + "expires_at", ExpiresAt = 4, +}); +st_fields_enum!(enum StContainerFenceFields { + "source_identity", SourceIdentity = 0, + "generation", Generation = 1, + "target_grant_revision", TargetGrantRevision = 2, + "target_set_hash", TargetSetHash = 3, + "allowed", Allowed = 4, +}); +st_fields_enum!(enum StConnectionAuthFields { + "connection_id", ConnectionId = 0, + "sender_identity", SenderIdentity = 1, + "call_auth_flags", CallAuthFlags = 2, +}); + +st_fields_enum!(enum StContainerEnvironmentFields { + "generation", Generation = 0, + "payload", Payload = 1, +}); + +/// Secret-bearing host state. Never expose its retained history through SQL or module syscalls. +#[derive(Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerEnvironmentRow { + pub generation: u64, + pub payload: Box<[u8]>, +} + +impl std::fmt::Debug for StContainerEnvironmentRow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StContainerEnvironmentRow") + .field("generation", &self.generation) + .field("payload", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentRow { + pub key: u8, + pub revision: Hash, + pub last_operation_id: u128, + pub payload: Box<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StPublishFenceRow { + pub key: u8, + pub publication_epoch: u64, + pub operation_id: u128, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentOperationRow { + pub operation_id: u128, + pub previous_revision: Option, + pub committed_revision: Hash, + pub commit_result: Box<[u8]>, + pub expires_at: TimestampViaI64, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerFenceRow { + pub source_identity: IdentityViaU256, + pub generation: u64, + pub target_grant_revision: u64, + pub target_set_hash: Hash, + pub allowed: bool, +} + +/// Captured host authentication for lifecycle cleanup, including crash recovery. +/// Only hosted connections need a row; absent rows retain ordinary flags zero. +/// JWT claims and sender equality never reconstruct these flags. +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StConnectionAuthRow { + pub connection_id: ConnectionIdViaU128, + pub sender_identity: IdentityViaU256, + pub call_auth_flags: u32, +} + +macro_rules! row_conversions { + ($($row:ty),+ $(,)?) => {$ ( + impl TryFrom> for $row { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } + } + impl From<$row> for ProductValue { + fn from(row: $row) -> Self { to_product_value(&row) } + } + )+}; +} + +row_conversions!( + StDeploymentRow, + StPublishFenceRow, + StDeploymentOperationRow, + StContainerFenceRow, + StConnectionAuthRow, + StContainerEnvironmentRow +); + +pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { + fn register(builder: &mut RawModuleDefV9Builder, name: &'static str) { + let ty = builder.add_type::(); + builder + .build_table(name, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); + } + register::(builder, ST_DEPLOYMENT_NAME); + register::(builder, ST_PUBLISH_FENCE_NAME); + register::(builder, ST_DEPLOYMENT_OPERATION_NAME); + register::(builder, ST_CONTAINER_FENCE_NAME); + register::(builder, ST_CONNECTION_AUTH_NAME); + register::(builder, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(super) fn validate_tables(def: &ModuleDef) { + validate_system_table::(def, ST_DEPLOYMENT_NAME); + validate_system_table::(def, ST_PUBLISH_FENCE_NAME); + validate_system_table::(def, ST_DEPLOYMENT_OPERATION_NAME); + validate_system_table::(def, ST_CONTAINER_FENCE_NAME); + validate_system_table::(def, ST_CONNECTION_AUTH_NAME); + validate_system_table::(def, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(crate) fn deployment_system_schemas() -> [TableSchema; 6] { + [ + st_schema(ST_DEPLOYMENT_NAME, ST_DEPLOYMENT_ID), + st_schema(ST_PUBLISH_FENCE_NAME, ST_PUBLISH_FENCE_ID), + st_schema(ST_DEPLOYMENT_OPERATION_NAME, ST_DEPLOYMENT_OPERATION_ID), + st_schema(ST_CONTAINER_FENCE_NAME, ST_CONTAINER_FENCE_ID), + st_schema(ST_CONNECTION_AUTH_NAME, ST_CONNECTION_AUTH_ID), + st_schema(ST_CONTAINER_ENVIRONMENT_NAME, ST_CONTAINER_ENVIRONMENT_ID), + ] +} + +pub(super) fn system_schema(table: TableId) -> Option { + let name = match table { + ST_DEPLOYMENT_ID => ST_DEPLOYMENT_NAME, + ST_PUBLISH_FENCE_ID => ST_PUBLISH_FENCE_NAME, + ST_DEPLOYMENT_OPERATION_ID => ST_DEPLOYMENT_OPERATION_NAME, + ST_CONTAINER_FENCE_ID => ST_CONTAINER_FENCE_NAME, + ST_CONNECTION_AUTH_ID => ST_CONNECTION_AUTH_NAME, + ST_CONTAINER_ENVIRONMENT_ID => ST_CONTAINER_ENVIRONMENT_NAME, + _ => return None, + }; + Some(st_schema(name, table)) +} + +/// These tables are read through dedicated host operations by module code. +/// In particular, resolving a numeric table or index ID must not bypass this. +pub(super) fn is_module_restricted_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +/// Snapshot history is exclusively available through authenticated host operations. +pub fn is_host_only_read_table(table: TableId) -> bool { + table == ST_CONTAINER_ENVIRONMENT_ID +} + +pub(super) fn is_module_restricted_deployment_index(index: IndexId) -> bool { + INDEXES.iter().any(|(_, restricted)| *restricted == index) +} + +/// Environment management has its own validated SQL path. Deployment and +/// authorization metadata may only be changed by authenticated host operations. +pub fn is_host_managed_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +pub(super) const CONSTRAINTS: [(&str, ConstraintId); 6] = [ + ("st_deployment_key_key", ConstraintId(27)), + ("st_publish_fence_key_key", ConstraintId(28)), + ("st_deployment_operation_operation_id_key", ConstraintId(29)), + ("st_container_fence_source_identity_key", ConstraintId(30)), + ("st_connection_auth_connection_id_key", ConstraintId(31)), + ("st_container_environment_generation_key", ConstraintId(32)), +]; + +pub(super) const INDEXES: [(&str, IndexId); 6] = [ + ("st_deployment_key_idx_btree", IndexId(31)), + ("st_publish_fence_key_idx_btree", IndexId(32)), + ("st_deployment_operation_operation_id_idx_btree", IndexId(33)), + ("st_container_fence_source_identity_idx_btree", IndexId(34)), + ("st_connection_auth_connection_id_idx_btree", IndexId(35)), + ("st_container_environment_generation_idx_btree", IndexId(36)), +]; diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs index d79f676edb8..4afd6e97820 100644 --- a/crates/datastore/src/system_tables/environment.rs +++ b/crates/datastore/src/system_tables/environment.rs @@ -35,8 +35,8 @@ pub(crate) fn st_env_schema() -> TableSchema { } /// Module code must use env_get even when it guesses numeric identifiers. pub fn is_module_restricted_table(table: TableId) -> bool { - table == ST_ENV_ID + table == ST_ENV_ID || super::deployment::is_module_restricted_deployment_table(table) } pub fn is_module_restricted_index(index: IndexId) -> bool { - index == IndexId(30) + index == IndexId(30) || super::deployment::is_module_restricted_deployment_index(index) } diff --git a/crates/engine/src/durability.rs b/crates/engine/src/durability.rs index 9c938a6401b..844da464467 100644 --- a/crates/engine/src/durability.rs +++ b/crates/engine/src/durability.rs @@ -1,17 +1,14 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use log::{error, info}; use spacetimedb_commitlog::payload::{ txdata::{Mutations, Ops}, Txdata, }; use spacetimedb_datastore::{execution_context::ReducerContext, traits::TxData}; use spacetimedb_durability::Transaction; -use spacetimedb_lib::Identity; use spacetimedb_sats::ProductValue; use crate::persistence::Durability; -use spacetimedb_runtime::Handle; pub(super) fn request_durability( durability: &Durability, @@ -32,21 +29,6 @@ pub(super) fn request_durability( })); } -pub(super) fn spawn_close(durability: Arc, runtime: &Handle, database_identity: Identity) { - let label = format!("[{database_identity}]"); - let runtime = runtime.clone(); - runtime.clone().spawn(async move { - match runtime.timeout(Duration::from_secs(10), durability.close()).await { - Err(_elapsed) => { - error!("{label} timeout waiting for durability shutdown"); - } - Ok(offset) => { - info!("{label} durability shut down at tx offset: {offset:?}"); - } - } - }); -} - fn prepare_tx_data_for_durability( tx_offset: u64, reducer_context: Option, diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs index 7c1a3e11129..6fb5a9260bf 100644 --- a/crates/engine/src/error.rs +++ b/crates/engine/src/error.rs @@ -97,6 +97,8 @@ pub enum ViewError { #[derive(Error, Debug)] pub enum DBError { + #[error("database storage is closed")] + DatabaseClosed, #[error("LibError: {0}")] Lib(#[from] LibError), #[error("BufferError: {0}")] diff --git a/crates/engine/src/hosted_admission.rs b/crates/engine/src/hosted_admission.rs new file mode 100644 index 00000000000..59b6f30ef1c --- /dev/null +++ b/crates/engine/src/hosted_admission.rs @@ -0,0 +1,282 @@ +//! Transient receiving-host admission, separate from durable generation fences. +//! +//! A replayed fence can be older than current control authority. Every database +//! open starts closed, including cold maintenance opens and restored databases. +//! Trusted platform code must reconcile the complete incoming fence inventory +//! before completing a sweep. This state is never serialized or restored. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use anyhow::ensure; +use parking_lot::Mutex; + +#[derive(Default)] +struct State { + revision: u64, + fence_revision: u64, + sweeping: bool, +} + +#[derive(Default)] +struct Inner { + open: AtomicBool, + state: Mutex, +} + +/// Owned by one live RelationalDB. Opening another copy of the same database +/// Identity creates an independent, closed gate. +#[derive(Default)] +pub struct HostedAdmission(Arc); + +impl HostedAdmission { + pub fn is_open(&self) -> bool { + self.0.open.load(Ordering::Acquire) + } + + /// Start one bounded reconciliation. A competing sweep fails immediately. + /// Dropping its ticket keeps admission closed and permits a later retry. + pub fn begin(&self) -> anyhow::Result { + let mut state = self.0.state.lock(); + ensure!(!state.sweeping, "hosted admission reconciliation is already running"); + self.0.open.store(false, Ordering::Release); + let revision = state + .revision + .checked_add(1) + .filter(|value| *value != u64::MAX) + .ok_or_else(|| anyhow::anyhow!("hosted admission revision exhausted"))?; + state.revision = revision; + state.sweeping = true; + Ok(HostedAdmissionSweep { + inner: self.0.clone(), + revision, + fence_revision: state.fence_revision, + }) + } + + /// Read while holding the database transaction that observes fence rows. + pub fn fence_revision(&self) -> u64 { + self.0.state.lock().fence_revision + } + + /// Call before changing a durable fence, within its mutation transaction. + /// Even a later rollback conservatively invalidates an inventory scan. A + /// delayed installation after startup closes admission again, so it cannot + /// insert an unexamined allowed row behind a completed scan's cursor. + pub fn fences_changed(&self) -> anyhow::Result<(u64, u64)> { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + let previous = state.fence_revision; + let Some(next) = previous.checked_add(1).filter(|next| *next != u64::MAX) else { + state.fence_revision = u64::MAX; + state.revision = u64::MAX; + state.sweeping = false; + anyhow::bail!("hosted fence revision exhausted"); + }; + state.fence_revision = next; + Ok((previous, next)) + } + + /// Invalidate any outstanding sweep. This prevents new admission; it is + /// not a substitute for a transactional fence and positive actor drainage. + pub fn close(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + + /// Permanently close a database whose storage writer is being shut down. + /// A retained handle cannot start a new sweep on that obsolete object. + pub fn seal(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = u64::MAX; + state.sweeping = false; + } +} + +/// A ticket is tied to the exact database-open state that issued it. Completing +/// one cannot open a replacement database, or undo a later close operation. +#[must_use = "dropping the sweep leaves hosted admission closed"] +pub struct HostedAdmissionSweep { + inner: Arc, + revision: u64, + fence_revision: u64, +} + +impl HostedAdmissionSweep { + /// Retain this guard in the async owner when moving the ticket to physical + /// blocking work. Cancelling that owner invalidates this exact sweep, even + /// when dropping the blocking task's JoinHandle cannot stop its execution. + pub fn cancellation_guard(&self) -> HostedAdmissionCancellation { + HostedAdmissionCancellation { + inner: Some(self.inner.clone()), + revision: self.revision, + } + } + + /// Trusted platform code calls this only after durable fence installation, + /// actor completion and current control confirmation of the entire sweep. + pub fn complete(self) -> anyhow::Result<()> { + let revision = self.fence_revision; + self.complete_with_fence_revision(revision) + } + + /// Complete a scan which made its own confirmed fence mutations. The + /// caller holds the actual DB transaction and has verified every stored + /// fence against current authority at this exact physical revision. + pub fn complete_with_fence_revision(self, fence_revision: u64) -> anyhow::Result<()> { + let mut state = self.inner.state.lock(); + ensure!( + state.sweeping && state.revision == self.revision, + "hosted admission reconciliation was invalidated" + ); + ensure!( + fence_revision != u64::MAX && state.fence_revision == fence_revision, + "hosted fence inventory changed during reconciliation" + ); + state.sweeping = false; + self.inner.open.store(true, Ordering::Release); + Ok(()) + } +} + +/// Cancellation belongs to one sweep revision. A late guard cannot close a +/// newer retry, and completion and cancellation serialize on the same mutex. +#[must_use = "retain until physical completion and disarm only after success"] +pub struct HostedAdmissionCancellation { + inner: Option>, + revision: u64, +} + +impl HostedAdmissionCancellation { + pub fn disarm(mut self) { + self.inner = None; + } +} + +impl Drop for HostedAdmissionCancellation { + fn drop(&mut self) { + let Some(inner) = &self.inner else { return }; + let mut state = inner.state.lock(); + if state.revision == self.revision { + inner.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + } +} + +impl Drop for HostedAdmissionSweep { + fn drop(&mut self) { + let mut state = self.inner.state.lock(); + if state.revision == self.revision { + state.sweeping = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancelled_sweep_retries_and_cannot_open_another_database() { + let original = HostedAdmission::default(); + let replacement = HostedAdmission::default(); + assert!(!original.is_open()); + let sweep = original.begin().unwrap(); + assert!(original.begin().is_err()); + drop(sweep); + assert!(!original.is_open()); + original.begin().unwrap().complete().unwrap(); + assert!(original.is_open()); + assert!(!replacement.is_open()); + } + + #[test] + fn stale_completion_and_drop_cannot_open_or_cancel_a_newer_sweep() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + gate.close(); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + assert!(!gate.is_open()); + assert!(gate.begin().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + gate.close(); + assert!(!gate.is_open()); + } + + #[test] + fn revision_exhaustion_remains_closed() { + let gate = HostedAdmission::default(); + gate.0.state.lock().revision = u64::MAX - 2; + gate.begin().unwrap().complete().unwrap(); + assert!(gate.is_open()); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + gate.close(); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + } + + #[test] + fn owner_cancellation_invalidates_detached_completion_without_closing_a_newer_retry() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + drop(cancellation); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + + gate.close(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + gate.close(); + gate.begin().unwrap().complete().unwrap(); + drop(cancellation); + assert!(gate.is_open()); + assert!(stale.complete().is_err()); + } + + #[test] + fn only_an_acknowledged_completion_survives_owner_drop() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + drop(cancellation); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + cancellation.disarm(); + assert!(gate.is_open()); + } + + #[test] + fn changed_fences_invalidate_scans_and_close_a_previously_open_gate() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (0, 1)); + assert!(ticket.complete().is_err()); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (1, 2)); + ticket.complete_with_fence_revision(2).unwrap(); + assert!(gate.is_open()); + gate.fences_changed().unwrap(); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + gate.0.state.lock().fence_revision = u64::MAX - 1; + assert!(gate.fences_changed().is_err()); + assert!(ticket.complete_with_fence_revision(u64::MAX).is_err()); + assert!(gate.begin().is_err()); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a38c80f3abb..07c4c4df7b5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,5 +1,6 @@ pub(crate) mod durability; pub mod error; +pub mod hosted_admission; pub mod metrics; pub mod persistence; pub mod relational_db; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 19cda5b6fbe..5500903412a 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -1,4 +1,4 @@ -use crate::durability::{request_durability, spawn_close as spawn_durability_close}; +use crate::durability::request_durability; use crate::error::{DBError, RestoreSnapshotError}; use crate::metrics::ExecutionCounters; use crate::metrics::ENGINE_METRICS; @@ -7,6 +7,8 @@ use crate::util::asyncify; use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; use spacetimedb_commitlog::repo::OnNewSegmentFn; use spacetimedb_commitlog::{self as commitlog, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; @@ -66,7 +68,9 @@ use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; use std::io; use std::ops::RangeBounds; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::OnceLock; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; @@ -102,8 +106,11 @@ pub type ConnectedClients = HashSet<(Identity, ConnectionId)>; pub struct RelationalDB { database_identity: Identity, owner_identity: Identity, + hosted_admission: super::hosted_admission::HostedAdmission, inner: Locking, + commits_closed: Arc, + shutdown: OnceLock>>>, durability: Option>, durability_runtime: Option, snapshot_worker: Option, @@ -142,9 +149,12 @@ impl std::fmt::Debug for RelationalDB { impl Drop for RelationalDB { fn drop(&mut self) { - // Attempt to flush the outstanding transactions. - if let (Some(durability), Some(runtime)) = (self.durability.take(), self.durability_runtime.take()) { - spawn_durability_close(durability, &runtime, self.database_identity); + self.hosted_admission.seal(); + if let Some(runtime) = &self.durability_runtime { + // Join the same owned close when a cancelled shutdown waiter was the + // last database owner. Never start a second provider close early. + let close = self.start_shutdown(runtime); + runtime.spawn(close); } } } @@ -164,12 +174,15 @@ impl RelationalDB { Self { inner, + commits_closed: Default::default(), + shutdown: Default::default(), durability, durability_runtime, snapshot_worker, database_identity, owner_identity, + hosted_admission: Default::default(), row_count_fn: default_row_count_fn(database_identity), disk_size_fn, @@ -194,6 +207,12 @@ impl RelationalDB { }); } + /// Transient container admission for this exact database open. The trusted + /// receiving host opens it only after reconciling current incoming fences. + pub fn hosted_admission(&self) -> &super::hosted_admission::HostedAdmission { + &self.hosted_admission + } + /// Open a database, which may or may not already exist. /// /// # Initialization @@ -364,23 +383,51 @@ impl RelationalDB { /// Shut down the database, without dropping it. /// - /// If the database is in-memory only, this does nothing. - /// Otherwise, it instructs the durability layer to shut down - /// and waits until all outstanding transactions are reported as durable. - /// - /// After calling this method, calling [Self::commit_tx_downgrade] or - /// [Self::commit_tx] will panic. - /// - /// Returns `None` if the database is in-memory only, - /// or nothing has been durably persisted yet. - /// - /// Returns the durable [TxOffset] in a `Some` otherwise. + /// Permanently closes hosted and commit admission, then joins the configured + /// durability writer. Transactions already holding the exclusive transaction + /// lock can finish before closure; later commits roll back with `DatabaseClosed`. + /// Caller cancellation does not cancel the physical close. pub async fn shutdown(&self) -> Option { - if let Some(durability) = &self.durability { - return durability.close().await; + self.hosted_admission.seal(); + match &self.durability_runtime { + Some(runtime) => self.start_shutdown(runtime).await, + None => self.start_shutdown(&Handle::tokio_current()).await, } + } - None + /// Close using the owning host runtime, including in-memory simulation databases. + pub async fn shutdown_with_runtime(&self, runtime: &Handle) -> Option { + self.hosted_admission.seal(); + self.start_shutdown(runtime).await + } + + fn start_shutdown(&self, runtime: &Handle) -> Shared>> { + self.shutdown + .get_or_init(|| { + let inner = self.inner.clone(); + let closed = self.commits_closed.clone(); + let durability = self.durability.clone(); + // A shutdown owns this task through both lock acquisition and actual + // writer completion. The blocking lock never occupies a runtime worker. + let close_runtime = runtime.clone(); + let task = runtime.spawn(async move { + close_runtime + .spawn_blocking(move || { + let tx = inner.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + closed.store(true, Ordering::Relaxed); + let _ = inner.rollback_mut_tx(tx); + }) + .await; + match durability { + Some(durability) => durability.close().await, + None => None, + } + }); + async move { task.await.expect("database writer close task panicked") } + .boxed() + .shared() + }) + .clone() } /// Create any system tables that are missing from the datastore. @@ -877,6 +924,13 @@ impl RelationalDB { tx: MutTx, ) -> Result, TxMetrics, Option)>, DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); // TODO: Never returns `None` -- should it? @@ -895,8 +949,15 @@ impl RelationalDB { } #[tracing::instrument(level = "trace", skip_all)] - pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> (Arc, TxMetrics, Tx) { + pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> Result<(Arc, TxMetrics, Tx), DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); let (tx_data, tx_metrics, tx, datastore_memory_bytes) = @@ -907,7 +968,7 @@ impl RelationalDB { self.maybe_do_snapshot(&tx_data); self.observe_datastore_memory(datastore_memory_bytes); - (tx_data, tx_metrics, tx) + Ok((tx_data, tx_metrics, tx)) } /// Get the [`DurableOffset`] of this database, or `None` if this is an @@ -1629,6 +1690,7 @@ impl RelationalDB { self.with_auto_commit(Workload::Internal, |mut_tx| { self.clear_all_views(mut_tx)?; self.clear_table(mut_tx, ST_CONNECTION_CREDENTIALS_ID)?; + self.clear_table(mut_tx, spacetimedb_datastore::system_tables::ST_CONNECTION_AUTH_ID)?; self.clear_table(mut_tx, ST_CLIENT_ID)?; self.clear_table(mut_tx, ST_VIEW_SUB_ID)?; Ok(()) @@ -2433,6 +2495,9 @@ pub mod tests_utils { } } +#[cfg(test)] +mod shutdown_tests; + #[cfg(test)] mod tests { #![allow(clippy::disallowed_macros)] diff --git a/crates/engine/src/relational_db/shutdown_tests.rs b/crates/engine/src/relational_db/shutdown_tests.rs new file mode 100644 index 00000000000..06c31b2fd83 --- /dev/null +++ b/crates/engine/src/relational_db/shutdown_tests.rs @@ -0,0 +1,65 @@ +use super::*; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use tests_utils::TestDB; + +#[test] +fn operation_drain_shutdown_serializes_transactions_and_survives_cancelled_waiter() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql); + insert(&mut tx, "BEFORE", "committed"); + let closing = tokio::spawn({ + let db = db.clone(); + async move { db.shutdown().await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while db.shutdown.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + closing.abort(); + assert!(closing.await.unwrap_err().is_cancelled()); + assert!(!db.commits_closed.load(Ordering::Relaxed)); + // This SQL/view transaction acquired the exclusive lock before shutdown. + let (_, _, read) = db.commit_tx_downgrade(tx, Workload::Sql).unwrap(); + let _ = db.release_tx(read); + assert!(db.shutdown().await.is_some()); + assert!(db.commits_closed.load(Ordering::Relaxed)); + for downgrade in [false, true] { + let mut late = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Subscribe); + insert(&mut late, "LATE", "must-roll-back"); + let error = if downgrade { + db.commit_tx_downgrade(late, Workload::Subscribe).err().unwrap() + } else { + db.commit_tx(late).err().unwrap() + }; + assert!(matches!(error, DBError::DatabaseClosed)); + } + let read = db.begin_tx(Workload::Internal); + assert_eq!(get(&read, "BEFORE").as_deref(), Some("committed")); + assert_eq!(get(&read, "LATE"), None); + let _ = db.release_tx(read); + }); +} + +fn insert(tx: &mut MutTx, key: &str, value: &str) { + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + ) + .unwrap(); +} + +fn get(tx: &Tx, key: &str) -> Option { + tx.iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into())) + .unwrap() + .next() + .map(|row| StEnvRow::try_from(row).unwrap().value) +} diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 2ea20b641a6..825bb0b5178 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -7,7 +7,11 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::mpsc, + future::{BoxFuture, Shared}, + FutureExt as _, StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -62,6 +66,7 @@ pub struct SnapshotWorker { snapshot_created: watch::Sender>, request_snapshot: mpsc::UnboundedSender, snapshot_repository: Arc, + completion: Shared>>>, } impl SnapshotWorker { @@ -89,12 +94,16 @@ impl SnapshotWorker { rt: rt.clone(), }), }; - rt.spawn(actor.run()); + let task = rt.spawn(actor.run()); + let completion = async move { task.await.map_err(|error| Arc::::from(error.to_string())) } + .boxed() + .shared(); Self { snapshot_created, request_snapshot: request_tx, snapshot_repository, + completion, } } @@ -103,6 +112,14 @@ impl SnapshotWorker { Self::new(snapshot_repository, compression, Handle::tokio_current()) } + /// Permanently close snapshot admission and join all previously accepted + /// snapshot and compression I/O. All clones observe the same completion. + /// Cancelling a waiter does not cancel the worker or its blocking I/O. + pub async fn shutdown(&self) -> Result<(), Arc> { + self.request_snapshot.close_channel(); + self.completion.clone().await + } + /// Finish the initialization of [Self] by passing a [SnapshotDatabaseState], /// or replace the current [SnapshotDatabaseState] with a new one. /// @@ -386,3 +403,63 @@ impl Compressor { } } } + +#[cfg(test)] +mod shutdown_tests { + use super::*; + use spacetimedb_datastore::{ + execution_context::Workload, + system_tables::{StEnvRow, ST_ENV_ID}, + traits::{IsolationLevel, MutTx as _}, + }; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + use spacetimedb_table::page_pool::PagePool; + + #[tokio::test(flavor = "multi_thread")] + async fn terminal_shutdown_drains_accepted_snapshot_after_waiter_cancellation() { + let dir = tempfile::tempdir().unwrap(); + let repository = Arc::new( + SnapshotRepository::open(SnapshotsPath::from_path_unchecked(dir.path()), Identity::ONE, 1).unwrap(), + ); + let datastore = Locking::bootstrap(Identity::ONE, PagePool::new_for_test()).unwrap(); + let mut tx = datastore.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "PERSISTED".into(), + value: "snapshot drain".into(), + }, + ) + .unwrap(); + datastore.commit_mut_tx(tx).unwrap(); + let worker = SnapshotWorker::new_tokio_current(repository.clone(), Compression::Enabled); + worker.set_state(datastore.committed_state.clone()); + let state = datastore.committed_state.write_arc(); + worker.request_snapshot(); + let mut first = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut first) + .await + .is_err()); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut second = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut second) + .await + .is_err()); + assert!(worker.request_snapshot.unbounded_send(Request::TakeSnapshot).is_err()); + assert_eq!(repository.latest_snapshot().unwrap(), None); + drop(state); + second.await.unwrap().unwrap(); + worker.shutdown().await.unwrap(); + assert_eq!(repository.latest_snapshot().unwrap(), Some(0)); + let snapshot = repository.read_snapshot(0, &PagePool::new_for_test()).unwrap(); + assert_eq!(snapshot.tx_offset, 0); + } +} diff --git a/crates/engine/src/sql/ast.rs b/crates/engine/src/sql/ast.rs index 892430ba1ea..4759c89e91d 100644 --- a/crates/engine/src/sql/ast.rs +++ b/crates/engine/src/sql/ast.rs @@ -34,6 +34,9 @@ impl SchemaView for SchemaViewer<'_, T> { } fn schema_for_table(&self, table_id: TableId) -> Option> { + if spacetimedb_datastore::system_tables::is_host_only_read_table(table_id) { + return None; + } self.tx .get_schema(table_id) .filter(|schema| self.auth.has_read_access(schema.table_access)) diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index b07774b3ee6..13d69140356 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -16,7 +16,7 @@ required-features = ["serde"] [features] default = ["serde", "metrics_impls"] -serde = ["dep:serde", "spacetimedb-sats/serde"] +serde = ["dep:serde", "dep:serde_json", "spacetimedb-sats/serde"] # Allows using `Arbitrary` impls defined in this crate. proptest = ["dep:proptest", "dep:proptest-derive"] # Allows using additional test methods. @@ -38,9 +38,11 @@ spacetimedb-memory-usage = { workspace = true, optional = true } spacetimedb-metrics = { workspace = true, optional = true } anyhow.workspace = true +thiserror.workspace = true derive_more.workspace = true hex.workspace = true serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } blake3.workspace = true enum-map = { workspace = true, optional = true } diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs new file mode 100644 index 00000000000..ba9b7949ab6 --- /dev/null +++ b/crates/lib/src/container.rs @@ -0,0 +1,482 @@ +//! Shared, versioned container deployment data. +//! +//! A decoded declaration is untrusted. Publication must normalize and validate +//! the complete effective deployment before hashing or admitting it. Runtime +//! capabilities and database authorization are additional admission checks. + +use crate::{bsatn, hash_bytes, Hash, SpacetimeType}; +use std::{collections::BTreeSet, fmt, str::FromStr}; + +#[cfg(feature = "serde")] +pub mod endpoints; +#[cfg(feature = "serde")] +pub mod exec; +#[cfg(feature = "serde")] +pub mod logs; +#[cfg(feature = "serde")] +pub mod operations; + +/// Version of the normalized deployment encoding, independent of module ABI. +pub const CONTAINER_SPEC_VERSION: u32 = 1; +pub const MAX_ARGV_ENTRIES: usize = 256; +pub const MAX_ENV_KEYS: usize = 256; +pub const MAX_PORTS: usize = 16; +pub const MAX_EXEC_STRING_BYTES: usize = 32 * 1024; +/// Includes NUL terminators, 64-bit pointer arrays, and reserved startup space. +pub const MAX_EXEC_BYTES: usize = 128 * 1024; +pub const EXEC_RESERVED_BYTES: usize = 4096; +pub const DEFAULT_STOP_GRACE_MS: u32 = 30_000; +pub const MAX_STOP_GRACE_MS: u32 = 120_000; + +/// The digest of an OCI object. This is never a SpacetimeDB Keccak-256 program key. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] +#[sats(crate = crate)] +pub enum OciDigest { + Sha256(Hash), +} + +impl OciDigest { + pub const fn sha256(bytes: [u8; 32]) -> Self { + Self::Sha256(Hash::from_byte_array(bytes)) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + match self { + Self::Sha256(hash) => &hash.data, + } + } +} + +impl fmt::Display for OciDigest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sha256(bytes) => write!(f, "sha256:{}", bytes.to_hex()), + } + } +} + +impl FromStr for OciDigest { + type Err = ContainerValidationError; + + fn from_str(value: &str) -> Result { + let hex = value + .strip_prefix("sha256:") + .ok_or_else(|| invalid("image_manifest", "only sha256 OCI digests are supported"))?; + if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(invalid( + "image_manifest", + "expected 64 lowercase hexadecimal digest characters", + )); + } + let mut bytes = [0; 32]; + hex::decode_to_slice(hex, &mut bytes).map_err(|_| invalid("image_manifest", "invalid SHA-256 digest"))?; + Ok(Self::sha256(bytes)) + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for OciDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for OciDigest { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ImagePlatform { + pub os: String, + pub architecture: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum ContainerMode { + Service, + Job, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum RestartPolicy { + Never, + OnFailure, + Always, +} + +impl RestartPolicy { + /// Only process termination drives this policy. Readiness is independent. + pub fn restarts_after(self, exit_code: Option) -> bool { + match self { + Self::Never => false, + Self::OnFailure => exit_code != Some(0), + Self::Always => true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerResources { + pub cpu_millicores: u64, + pub memory_bytes: u64, + pub scratch_bytes: u64, + /// Linux tasks, including threads and commands started through exec. + pub pids_max: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortProtocol { + Http, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortExposure { + Public, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ReadinessProbe { + Tcp(TcpProbe), + Http(HttpProbe), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct TcpProbe { + pub timeout_ms: u32, + pub interval_ms: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct HttpProbe { + pub path: String, + pub timeout_ms: u32, + pub interval_ms: u32, +} + +impl Default for ReadinessProbe { + fn default() -> Self { + Self::Tcp(TcpProbe { + timeout_ms: 1000, + interval_ms: 5000, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerPort { + pub name: String, + pub port: u16, + pub protocol: PortProtocol, + /// Required in the public input, with no implicit exposure default. + pub exposure: PortExposure, + #[cfg_attr(feature = "serde", serde(default))] + pub readiness_probe: ReadinessProbe, +} + +/// A placeholder declaration is never admitted until the mount protocol ships. +/// Keeping explicit declarations lets Stage 1 return an unsupported-feature error. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerMount { + pub database: String, + pub source: String, + pub target: String, + pub read_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerSpec { + pub image_manifest: OciDigest, + pub image_platform: ImagePlatform, + /// Effective complete argv after applying OCI Entrypoint/Cmd or override. + pub argv: Vec, + pub user: String, + pub working_directory: String, + pub mode: ContainerMode, + pub restart: RestartPolicy, + pub env_keys: Vec, + pub resources: ContainerResources, + pub ports: Vec, + pub mounts: Vec, + pub stop_grace_ms: u32, +} + +/// Explicit component removal is distinct from omission or an empty replacement. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +#[expect( + clippy::large_enum_variant, + reason = "the normalized publish request owns its single container spec" +)] +pub enum ContainerAction { + #[default] + Keep, + Set(ContainerSpec), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container {field}: {reason}")] +pub struct ContainerValidationError { + pub field: &'static str, + pub reason: &'static str, +} + +fn invalid(field: &'static str, reason: &'static str) -> ContainerValidationError { + ContainerValidationError { field, reason } +} + +/// Limits are operational configuration; normalized specs are checked again at +/// admission against the selected node's enforceable capacities and capabilities. +#[derive(Clone, Copy, Debug)] +pub struct ContainerSpecLimits { + pub resources: ContainerResources, +} + +impl Default for ContainerSpecLimits { + fn default() -> Self { + Self { + resources: ContainerResources { + cpu_millicores: 64_000, + memory_bytes: 128 * 1024 * 1024 * 1024, + scratch_bytes: 1024 * 1024 * 1024 * 1024, + pids_max: 4096, + }, + } + } +} + +impl ContainerSpec { + /// Sort semantically unordered declarations so equivalent specs hash alike. + /// Duplicates are rejected, never silently deduplicated. + pub fn normalize(mut self, limits: &ContainerSpecLimits) -> Result { + self.validate(limits)?; + self.env_keys.sort_unstable(); + self.ports.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(self) + } + + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), ContainerValidationError> { + if self.image_platform.os != "linux" || !matches!(self.image_platform.architecture.as_str(), "amd64" | "arm64") + { + return Err(invalid("image_platform", "expected linux/amd64 or linux/arm64")); + } + if !self.mounts.is_empty() { + return Err(invalid("mounts", "SpacetimeFS mounts are not supported by Stage 1")); + } + if self.mode == ContainerMode::Job && self.restart == RestartPolicy::Always { + return Err(invalid("restart", "jobs cannot use the always restart policy")); + } + if self.argv.is_empty() || self.argv.len() > MAX_ARGV_ENTRIES || self.argv[0].is_empty() { + return Err(invalid( + "argv", + "a nonempty command with at most 256 arguments is required", + )); + } + validate_exec_size(&self.argv, &[])?; + if self.user.len() > 255 || self.user.bytes().any(|b| b == 0 || b.is_ascii_control()) { + return Err(invalid( + "user", + "user must fit 255 bytes and contain no control characters", + )); + } + if !self.working_directory.starts_with('/') + || self.working_directory.len() > 4096 + || self.working_directory.contains('\0') + { + return Err(invalid( + "working_directory", + "expected an absolute Linux path of at most 4096 bytes", + )); + } + if self.stop_grace_ms > MAX_STOP_GRACE_MS { + return Err(invalid("stop_grace_ms", "stop grace exceeds the supported deadline")); + } + let r = self.resources; + let max = limits.resources; + if r.cpu_millicores == 0 + || r.cpu_millicores > max.cpu_millicores + || r.memory_bytes == 0 + || r.memory_bytes > max.memory_bytes + || r.scratch_bytes == 0 + || r.scratch_bytes > max.scratch_bytes + || r.pids_max == 0 + || r.pids_max > max.pids_max + { + return Err(invalid( + "resources", + "resource reservations must be positive and within server limits", + )); + } + if self.env_keys.len() > MAX_ENV_KEYS { + return Err(invalid("env_keys", "too many environment keys")); + } + let mut keys = BTreeSet::new(); + for key in &self.env_keys { + validate_env_key(key)?; + if !keys.insert(key) { + return Err(invalid("env_keys", "duplicate environment key")); + } + } + if self.ports.len() > MAX_PORTS { + return Err(invalid("ports", "too many declared ports")); + } + let (mut names, mut numbers) = (BTreeSet::new(), BTreeSet::new()); + for port in &self.ports { + let bytes = port.name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 32 + || !bytes[0].is_ascii_lowercase() + || !bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-') + { + return Err(invalid("ports.name", "expected [a-z][a-z0-9-]{0,31}")); + } + if port.port == 0 || !names.insert(&port.name) || !numbers.insert(port.port) { + return Err(invalid("ports", "port names and nonzero port numbers must be unique")); + } + let (timeout, interval) = match &port.readiness_probe { + ReadinessProbe::Tcp(TcpProbe { + timeout_ms, + interval_ms, + }) => (*timeout_ms, *interval_ms), + ReadinessProbe::Http(HttpProbe { + path, + timeout_ms, + interval_ms, + }) => { + if !path.starts_with('/') + || path.starts_with("//") + || path.len() > 2048 + || path + .bytes() + .any(|b| b.is_ascii_control() || b == b' ' || b == b'\\' || b == b'#') + { + return Err(invalid( + "readiness_probe.path", + "expected a bounded origin-relative HTTP path", + )); + } + (*timeout_ms, *interval_ms) + } + }; + if timeout == 0 || timeout > 30_000 || interval == 0 || interval > 300_000 || timeout > interval { + return Err(invalid("readiness_probe", "invalid probe timeout or interval")); + } + } + Ok(()) + } + + /// Domain-separated, versioned BSATN encoding, after normalization. + /// This hashes the container spec only; the full deployment also includes + /// the module selection and uses its own revision domain. + pub fn canonical_hash(&self, limits: &ContainerSpecLimits) -> Result { + let normalized = self.clone().normalize(limits)?; + let encoded = bsatn::to_vec(&(CONTAINER_SPEC_VERSION, normalized)) + .expect("encoding an in-memory container specification cannot fail"); + let mut bytes = b"spacetimedb/container-spec\0".to_vec(); + bytes.extend(encoded); + Ok(hash_bytes(&bytes)) + } +} + +pub fn validate_env_key(key: &str) -> Result<(), ContainerValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > 256 + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(invalid("env_keys", "invalid POSIX environment variable name")); + } + if key.starts_with("SPACETIMEDB_") { + return Err(invalid( + "env_keys", + "SPACETIMEDB_ variables are reserved for the platform", + )); + } + Ok(()) +} + +/// Call after merging image environment, the database snapshot, and platform +/// variables. Error messages deliberately contain neither argv nor env values. +pub fn validate_exec_size(argv: &[String], env: &[String]) -> Result<(), ContainerValidationError> { + let count = argv + .len() + .checked_add(env.len()) + .and_then(|n| n.checked_add(2)) + .ok_or_else(|| invalid("exec", "too many arguments or environment entries"))?; + let mut total = count + .checked_mul(8) + .and_then(|n| n.checked_add(EXEC_RESERVED_BYTES)) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + for value in argv.iter().chain(env) { + if value.contains('\0') || value.len() >= MAX_EXEC_STRING_BYTES { + return Err(invalid( + "exec", + "argument or environment entry contains NUL or exceeds the per-entry limit", + )); + } + total = total + .checked_add(value.len() + 1) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + } + if total > MAX_EXEC_BYTES { + return Err(invalid( + "exec", + "combined argument and environment size exceeds the startup limit", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/endpoints.rs b/crates/lib/src/container/endpoints.rs new file mode 100644 index 00000000000..d5ab7964b11 --- /dev/null +++ b/crates/lib/src/container/endpoints.rs @@ -0,0 +1,57 @@ +//! Public address discovery. These addresses carry no administrative or runtime +//! authority and do not promise that an application is currently ready. + +use super::PortProtocol; +use crate::Identity; +use serde::{Deserialize, Serialize}; + +pub const ENDPOINTS_PENDING: &str = "endpoints_pending"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoints { + pub database_identity: Identity, + pub endpoints: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoint { + pub name: String, + pub protocol: PortProtocol, + pub url: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_wire_contains_only_explicit_public_addresses() { + let response = ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + }; + let encoded = serde_json::to_value(&response).unwrap(); + assert_eq!(encoded.as_object().unwrap().len(), 2); + assert_eq!( + encoded["endpoints"][0], + serde_json::json!({ + "name": "http", + "protocol": "http", + "url": "https://aaaqeayeaudaocajbifqydiob4.container.example.net/" + }) + ); + assert_eq!( + serde_json::from_value::(encoded.clone()).unwrap(), + response + ); + let mut changed = encoded; + changed["endpoints"][0]["upstream"] = serde_json::json!("10.0.0.1:8080"); + assert!(serde_json::from_value::(changed).is_err()); + } +} diff --git a/crates/lib/src/container/exec.rs b/crates/lib/src/container/exec.rs new file mode 100644 index 00000000000..f517e8a79d6 --- /dev/null +++ b/crates/lib/src/container/exec.rs @@ -0,0 +1,226 @@ +//! Versioned messages for one authenticated WebSocket per container exec. +//! +//! Authenticate the upgrade before accepting a Start message. Start must be the +//! first application message and is accepted only once. A Ready response binds +//! the socket to that exact database, generation and session until Exit/Error. +//! Reconnection or replay must never silently start another process. EOF closes +//! only stdin; closing the socket does not establish process termination. +//! Reject duplicate Start, stdin after EOF, and controls after the terminal +//! response. Output precedes exactly one Exit/Error. Bound queued bytes as well +//! as individual messages. Retain the chosen RuntimeHandle throughout the +//! session, even if another instance starts under the same deployment. +//! +//! These codecs bound and validate messages, not authority or session ordering. +//! The server separately checks Admin permission, the complete current runtime +//! binding and its lease, and validates the inherited environment before exec. + +use super::{ + operations::{decimal_u64, ContainerErrorCode}, + validate_env_key, validate_exec_size, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, +}; +use crate::{deployment::uuid_json, Identity, Uuid}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, fmt}; + +pub const SUBPROTOCOL: &str = "v1.container.exec.spacetimedb"; +/// JSON escapes may expand a valid 128 KiB argv/environment by up to six times. +pub const MAX_CONTROL_BYTES: usize = 1024 * 1024; +pub const MAX_DATA_BYTES: usize = 64 * 1024; +pub const MAX_BINARY_BYTES: usize = MAX_DATA_BYTES + 1; + +/// Fixed diagnostics never reflect argv, environment values or peer input. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container exec message")] +pub struct ProtocolError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TerminalSize { + pub rows: u16, + pub columns: u16, +} +impl TerminalSize { + pub fn validate(self) -> Result<(), ProtocolError> { + if (1..=4096).contains(&self.rows) && (1..=4096).contains(&self.columns) { + Ok(()) + } else { + Err(ProtocolError) + } + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecStart { + /// Exact control generation selected by status, never a floating "current". + #[serde(with = "decimal_u64")] + pub generation: u64, + /// Literal argv; the server never supplies an implicit shell or user override. + pub argv: Vec, + /// None inherits the container's directory. Some must be an absolute path. + pub working_directory: Option, + /// Overrides for this process only; platform keys remain reserved. + pub environment: BTreeMap, + pub stdin: bool, + /// Some allocates a PTY at this initial size; None keeps stdout/stderr separate. + pub terminal: Option, +} +impl fmt::Debug for ExecStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecStart") + .field("generation", &self.generation) + .field("stdin", &self.stdin) + .field("terminal", &self.terminal) + .finish_non_exhaustive() + } +} +impl ExecStart { + /// The backend must also count its inherited environment after applying + /// these overrides. This validation cannot see that immutable launch state. + pub fn validate(&self) -> Result<(), ProtocolError> { + if self.generation == 0 + || self.argv.is_empty() + || self.argv[0].is_empty() + || self.argv.len() > MAX_ARGV_ENTRIES + || self.environment.len() > MAX_ENV_KEYS + || self.environment.keys().any(|key| validate_env_key(key).is_err()) + || self + .working_directory + .as_ref() + .is_some_and(|path| !path.starts_with('/') || path.len() > 4096 || path.contains('\0')) + { + return Err(ProtocolError); + } + if let Some(size) = self.terminal { + size.validate()?; + } + let environment: Vec<_> = self + .environment + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect(); + validate_exec_size(&self.argv, &environment).map_err(|_| ProtocolError) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case", deny_unknown_fields)] +pub enum ClientControl { + Start(ExecStart), + StdinEof, + Resize(TerminalSize), + /// Linux signal number on the supported x86_64/aarch64 guest platforms. + /// Targets this exec process, never the container main process or a host PID. + Signal(u8), +} +impl ClientControl { + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_CONTROL_BYTES { + return Err(ProtocolError); + } + let message: Self = serde_json::from_slice(bytes).map_err(|_| ProtocolError)?; + message.validate()?; + Ok(message) + } + pub fn validate(&self) -> Result<(), ProtocolError> { + match self { + Self::Start(start) => start.validate(), + Self::Resize(size) => size.validate(), + Self::Signal(1..=64) | Self::StdinEof => Ok(()), + Self::Signal(_) => Err(ProtocolError), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecReady { + pub database_identity: Identity, + #[serde(with = "decimal_u64")] + pub generation: u64, + #[serde(with = "uuid_json")] + pub session_id: Uuid, + pub tty: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case", deny_unknown_fields)] +pub enum ServerControl { + Ready(ExecReady), + /// Follows the final output frame and confirms observed process exit. + Exit { + exit_code: i64, + }, + /// Terminal failure. Does not assert that an already started process exited. + Error { + error: ContainerErrorCode, + }, +} +impl ServerControl { + pub fn decode(bytes: &[u8]) -> Result { + // Responses contain metadata only, so they need much less space than Start. + if bytes.len() > 4096 { + return Err(ProtocolError); + } + let message: Self = serde_json::from_slice(bytes).map_err(|_| ProtocolError)?; + if matches!(&message, Self::Ready(ready) if ready.generation == 0 || ready.session_id.as_u128() == 0) { + return Err(ProtocolError); + } + Ok(message) + } +} + +/// PTY output uses Stdout because the guest PTY combines both output streams. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Stream { + Stdin = 0, + Stdout = 1, + Stderr = 2, +} + +/// One channel byte followed by arbitrary nonempty process bytes. Binary data +/// need not be UTF-8; empty messages never stand in for an EOF control message. +pub fn encode_data(stream: Stream, bytes: &[u8]) -> Result, ProtocolError> { + if bytes.is_empty() || bytes.len() > MAX_DATA_BYTES { + return Err(ProtocolError); + } + let mut frame = Vec::with_capacity(bytes.len() + 1); + frame.push(stream as u8); + frame.extend_from_slice(bytes); + Ok(frame) +} + +pub fn decode_stdin(frame: &[u8]) -> Result<&[u8], ProtocolError> { + let (stream, data) = decode_data(frame)?; + if stream == Stream::Stdin { + Ok(data) + } else { + Err(ProtocolError) + } +} + +pub fn decode_output(frame: &[u8]) -> Result<(Stream, &[u8]), ProtocolError> { + let (stream, data) = decode_data(frame)?; + if stream == Stream::Stdin { + Err(ProtocolError) + } else { + Ok((stream, data)) + } +} + +fn decode_data(frame: &[u8]) -> Result<(Stream, &[u8]), ProtocolError> { + if frame.len() < 2 || frame.len() > MAX_BINARY_BYTES { + return Err(ProtocolError); + } + let stream = match frame[0] { + 0 => Stream::Stdin, + 1 => Stream::Stdout, + 2 => Stream::Stderr, + _ => return Err(ProtocolError), + }; + Ok((stream, &frame[1..])) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/exec/tests.rs b/crates/lib/src/container/exec/tests.rs new file mode 100644 index 00000000000..e3d1ff496e6 --- /dev/null +++ b/crates/lib/src/container/exec/tests.rs @@ -0,0 +1,184 @@ +use super::*; + +fn start() -> ExecStart { + ExecStart { + generation: u64::MAX, + argv: vec!["/bin/echo".into(), "secret-argv; $(literal)".into()], + working_directory: Some("/secret-directory".into()), + environment: BTreeMap::from([("TOKEN".into(), "secret-value".into())]), + stdin: true, + terminal: None, + } +} + +#[test] +fn start_preserves_literal_arguments_exact_generation_and_redacts_debug() { + let message = ClientControl::Start(start()); + let json = serde_json::to_vec(&message).unwrap(); + assert!(String::from_utf8_lossy(&json).contains("\"generation\":\"18446744073709551615\"")); + assert_eq!(ClientControl::decode(&json).unwrap(), message); + let debug = format!("{message:?}"); + for secret in ["secret-argv", "literal", "secret-directory", "TOKEN", "secret-value"] { + assert!(!debug.contains(secret)); + } +} + +#[test] +fn invalid_start_never_reflects_values_and_rejects_ambiguous_generation() { + let baseline = serde_json::to_value(ClientControl::Start(start())).unwrap(); + for value in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("0"), + serde_json::json!("-1"), + ] { + let mut changed = baseline.clone(); + changed["data"]["generation"] = value; + assert!(ClientControl::decode(&serde_json::to_vec(&changed).unwrap()).is_err()); + } + let changes: [fn(&mut ExecStart); 12] = [ + |s| s.argv.clear(), + |s| s.argv[0].clear(), + |s| s.argv.push("secret-value\0".into()), + |s| s.argv = vec!["x".into(); MAX_ARGV_ENTRIES + 1], + |s| s.working_directory = Some("relative-secret-path".into()), + |s| s.working_directory = Some("/secret\0".into()), + |s| s.working_directory = Some(format!("/{}", "x".repeat(4096))), + |s| { + s.environment.insert("SPACETIMEDB_TOKEN".into(), "secret-value".into()); + }, + |s| { + s.environment.insert("INVALID=KEY".into(), "secret-value".into()); + }, + |s| { + s.environment.insert("TOKEN".into(), "secret\0".into()); + }, + |s| s.environment = (0..=MAX_ENV_KEYS).map(|i| (format!("E{i}"), String::new())).collect(), + |s| s.terminal = Some(TerminalSize { rows: 0, columns: 80 }), + ]; + for change in changes { + let mut changed = start(); + change(&mut changed); + let bytes = serde_json::to_vec(&ClientControl::Start(changed)).unwrap(); + let error = ClientControl::decode(&bytes).unwrap_err(); + assert_eq!(error.to_string(), "invalid container exec message"); + assert!(!format!("{error:?}").contains("secret")); + } +} + +#[test] +fn aggregate_budget_counts_overrides_and_json_bound_accepts_escaped_values() { + let mut request = start(); + request.environment.clear(); + request.argv = vec!["program".into()]; + request.argv.extend(vec!["\u{1}".repeat(15_000); 8]); + let bytes = serde_json::to_vec(&ClientControl::Start(request.clone())).unwrap(); + assert!(bytes.len() > 700_000 && bytes.len() < MAX_CONTROL_BYTES); + assert!(ClientControl::decode(&bytes).is_ok()); + request.environment.insert("TOKEN".into(), "x".repeat(15_000)); + assert!(request.validate().is_err()); + request = start(); + request + .environment + .insert("TOKEN".into(), "x".repeat(super::super::MAX_EXEC_STRING_BYTES)); + assert!(request.validate().is_err()); + assert!(ClientControl::decode(&vec![b' '; MAX_CONTROL_BYTES + 1]).is_err()); +} + +#[test] +fn binary_frames_preserve_arbitrary_bytes_enforce_direction_and_bound_size() { + let bytes = [0, 255, 128, b'\n']; + let input = encode_data(Stream::Stdin, &bytes).unwrap(); + assert_eq!(decode_stdin(&input).unwrap(), bytes); + assert!(decode_output(&input).is_err()); + for stream in [Stream::Stdout, Stream::Stderr] { + let output = encode_data(stream, &bytes).unwrap(); + assert_eq!(decode_output(&output).unwrap(), (stream, bytes.as_slice())); + assert!(decode_stdin(&output).is_err()); + } + let max = encode_data(Stream::Stdin, &vec![0xff; MAX_DATA_BYTES]).unwrap(); + assert_eq!(max.len(), MAX_BINARY_BYTES); + assert_eq!(decode_stdin(&max).unwrap().len(), MAX_DATA_BYTES); + for invalid in [vec![], vec![0], vec![3, 1], vec![0; MAX_BINARY_BYTES + 1]] { + assert!(decode_stdin(&invalid).is_err()); + assert!(decode_output(&invalid).is_err()); + } + assert!(encode_data(Stream::Stdin, &[]).is_err()); + assert!(encode_data(Stream::Stdout, &vec![0; MAX_DATA_BYTES + 1]).is_err()); +} + +#[test] +fn eof_resize_and_signal_controls_are_distinct_and_checked() { + let eof = br#"{"type":"stdin_eof"}"#; + assert_eq!(ClientControl::decode(eof).unwrap(), ClientControl::StdinEof); + assert!(decode_stdin(&[0]).is_err()); + for control in [ + ClientControl::Resize(TerminalSize { rows: 24, columns: 80 }), + ClientControl::Signal(1), + ClientControl::Signal(64), + ] { + assert_eq!( + ClientControl::decode(&serde_json::to_vec(&control).unwrap()).unwrap(), + control + ); + } + for control in [ + ClientControl::Resize(TerminalSize { + rows: 4097, + columns: 80, + }), + ClientControl::Signal(0), + ClientControl::Signal(65), + ] { + assert!(ClientControl::decode(&serde_json::to_vec(&control).unwrap()).is_err()); + } +} + +#[test] +fn protocol_rejects_unknown_duplicate_and_trailing_fields_without_echo() { + for bad in [ + br#"{"type":"signal","data":2,"secret":"value"}"#.as_slice(), + br#"{"type":"resize","data":{"rows":24,"columns":80,"secret":"value"}}"#, + br#"{"type":"signal","type":"stdin_eof","data":2}"#, + br#"{"type":"signal","data":2} {"secret":"value"}"#, + br#"{"type":"not-a-command","data":"secret-value"}"#, + ] { + assert_eq!(ClientControl::decode(bad).unwrap_err(), ProtocolError); + } +} + +#[test] +fn server_messages_keep_exact_session_metadata_and_reject_invalid_ready() { + let ready = ExecReady { + database_identity: Identity::from_byte_array([7; 32]), + generation: u64::MAX, + session_id: Uuid::from_u128(42), + tty: true, + }; + for message in [ + ServerControl::Ready(ready.clone()), + ServerControl::Exit { exit_code: 137 }, + ServerControl::Error { + error: ContainerErrorCode::AccessDenied, + }, + ] { + assert_eq!( + ServerControl::decode(&serde_json::to_vec(&message).unwrap()).unwrap(), + message + ); + } + for changed in [ + ExecReady { + generation: 0, + ..ready.clone() + }, + ExecReady { + session_id: Uuid::from_u128(0), + ..ready + }, + ] { + assert!(ServerControl::decode(&serde_json::to_vec(&ServerControl::Ready(changed)).unwrap()).is_err()); + } + assert!(ServerControl::decode(br#"{"type":"exit","data":{"exit_code":0,"secret":"value"}}"#).is_err()); + assert!(ServerControl::decode(&vec![b' '; 4097]).is_err()); +} diff --git a/crates/lib/src/container/logs.rs b/crates/lib/src/container/logs.rs new file mode 100644 index 00000000000..4e9bf7e973f --- /dev/null +++ b/crates/lib/src/container/logs.rs @@ -0,0 +1,195 @@ +//! Bounded process-log pages. One selection follows one exact attempt and +//! capture; it never silently switches to a replacement container. + +use super::operations::decimal_u64; +use crate::{deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +pub const MAX_LOG_PAGE_BYTES: usize = 512 * 1024; +pub const MAX_LOG_PAGE_RECORDS: usize = 64; +pub const MAX_LOG_RECORD_BYTES: usize = 8192; +pub const MAX_LOG_CURSOR_BYTES: usize = 512; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerLogQuery { + #[serde(default, with = "optional_decimal_u64")] + pub generation: Option, + pub cursor: Option, + #[serde(default)] + pub follow: bool, +} + +impl ContainerLogQuery { + pub fn validate(&self) -> Result<(), &'static str> { + if self.generation == Some(0) + || self + .cursor + .as_ref() + .is_some_and(|cursor| !valid_cursor(cursor) || self.generation.is_none()) + { + return Err("invalid container log selection"); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogGapReason { + SourceRotation, + SourceBackpressure, + CaptureStartedLate, +} + +/// No Debug implementation: process output must not enter platform diagnostics. +/// bytes preserves arbitrary output, including binary data and partial lines. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogEvent { + Data { + #[serde(with = "decimal_i64")] + timestamp_micros: i64, + stream: LogStream, + bytes: Vec, + }, + Gap { + #[serde(with = "decimal_i64")] + timestamp_micros: i64, + reason: LogGapReason, + #[serde(with = "optional_decimal_u64")] + missed_bytes: Option, + }, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LogRecord { + #[serde(with = "decimal_u64")] + pub sequence: u64, + pub event: LogEvent, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogEnd { + Eof, + Cancelled, + SourceFailed, + OwnerLost, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogLoss { + SourceUnavailable, + SourceOpenFailed, + SourceBindingMismatch, + SourceRejected, + AttachmentTimeout, + DrainFailed, + DrainTimeout, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerLogPage { + pub database_identity: Identity, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub deployment_revision: Hash, + #[serde(with = "uuid_json")] + pub publication_operation: Uuid, + #[serde(with = "decimal_u64")] + pub publication_epoch: u64, + #[serde(with = "uuid_json")] + pub capture_id: Uuid, + pub records: Vec, + pub next_cursor: String, + #[serde(with = "decimal_u64")] + pub oldest_retained_sequence: u64, + pub retention_gap: bool, + /// More records were available when this page was read. + pub has_more: bool, + /// Present only on the page reaching the final record. A loss observation + /// remains incomplete even if the Docker source also reached natural EOF. + pub end: Option, + pub loss: Option, +} + +impl ContainerLogPage { + pub fn validate(&self) -> Result<(), &'static str> { + if self.generation == 0 + || self.publication_epoch == 0 + || self.capture_id.as_u128() == 0 + || self.publication_operation.as_u128() == 0 + || self.oldest_retained_sequence == 0 + || !valid_cursor(&self.next_cursor) + || self.records.len() > MAX_LOG_PAGE_RECORDS + || (self.has_more && (self.records.is_empty() || self.end.is_some())) + { + return Err("invalid container log page"); + } + let mut previous = 0; + for record in &self.records { + if record.sequence < self.oldest_retained_sequence + || record.sequence <= previous + || matches!(&record.event, LogEvent::Data { bytes, .. } if bytes.len() > MAX_LOG_RECORD_BYTES) + { + return Err("invalid container log record"); + } + previous = record.sequence; + } + Ok(()) + } +} + +fn valid_cursor(cursor: &str) -> bool { + (1..=MAX_LOG_CURSOR_BYTES).contains(&cursor.len()) + && cursor + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +mod decimal_i64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &i64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: i64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal i64")); + } + Ok(value) + } +} + +mod optional_decimal_u64 { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + pub fn serialize(number: &Option, serializer: S) -> Result { + number.map(|value| value.to_string()).serialize(serializer) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Option::::deserialize(deserializer)? + .map(|text| { + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + }) + .transpose() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/logs/tests.rs b/crates/lib/src/container/logs/tests.rs new file mode 100644 index 00000000000..f31c0c50894 --- /dev/null +++ b/crates/lib/src/container/logs/tests.rs @@ -0,0 +1,72 @@ +use super::*; + +fn page() -> ContainerLogPage { + ContainerLogPage { + database_identity: Identity::ONE, + generation: u64::MAX, + deployment_revision: Hash::from_byte_array([1; 32]), + publication_operation: Uuid::from_u128(1), + publication_epoch: u64::MAX, + capture_id: Uuid::from_u128(2), + records: vec![LogRecord { + sequence: u64::MAX, + event: LogEvent::Data { + timestamp_micros: i64::MIN, + stream: LogStream::Stderr, + bytes: vec![0, 255, b'\n'], + }, + }], + next_cursor: "encoded_cursor".into(), + oldest_retained_sequence: u64::MAX, + retention_gap: true, + has_more: false, + end: Some(LogEnd::Eof), + loss: Some(LogLoss::DrainTimeout), + } +} + +#[test] +fn pages_preserve_binary_output_and_exact_browser_integers() { + let expected = page(); + expected.validate().unwrap(); + let value = serde_json::to_value(&expected).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!(value["records"][0]["event"]["timestamp_micros"], i64::MIN.to_string()); + let decoded: ContainerLogPage = serde_json::from_value(value).unwrap(); + assert!(decoded == expected); + assert_eq!(decoded.end, Some(LogEnd::Eof)); + assert_eq!(decoded.loss, Some(LogLoss::DrainTimeout)); +} + +#[test] +fn malformed_or_lossy_integer_and_capture_selections_are_rejected() { + for generation in [ + serde_json::json!(u64::MAX), + serde_json::json!("01"), + serde_json::json!("-1"), + ] { + let mut value = serde_json::to_value(page()).unwrap(); + value["generation"] = generation; + assert!(serde_json::from_value::(value).is_err()); + } + for cursor in ["", "has space", "../other"] { + let query = ContainerLogQuery { + generation: Some(1), + cursor: Some(cursor.into()), + follow: true, + }; + assert!(query.validate().is_err()); + } + assert!(ContainerLogQuery { + generation: None, + cursor: Some("encoded".into()), + follow: true + } + .validate() + .is_err()); + let mut oversized = page(); + if let LogEvent::Data { bytes, .. } = &mut oversized.records[0].event { + bytes.resize(MAX_LOG_RECORD_BYTES + 1, 0); + } + assert!(oversized.validate().is_err()); +} diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs new file mode 100644 index 00000000000..5c1d6b7118f --- /dev/null +++ b/crates/lib/src/container/operations.rs @@ -0,0 +1,208 @@ +//! Control-plane container inspection and idempotent lifecycle messages. +use crate::{container::endpoints::ContainerEndpoint, deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +mod resources; +pub use resources::{ContainerConfiguration, ReportedUsage, ResourceLimits, UsageTotals}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerAction { + Start, + Stop, + Restart, +} +impl ContainerAction { + pub const fn path(self) -> &'static str { + match self { + Self::Start => "start", + Self::Stop => "stop", + Self::Restart => "restart", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationRequest { + #[serde(with = "uuid_json")] + pub request_id: Uuid, +} + +/// Confirms admission only. It does not claim physical stop or runtime readiness. +/// An exact retry returns the generation originally recorded for this request. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationReceipt { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub request_id: Uuid, + pub action: ContainerAction, + #[serde(with = "decimal_u64")] + pub generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DesiredState { + Stopped, + Running, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObservedState { + Pending, + Starting, + Running, + Ready, + Draining, + Stopped, + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Condition { + None, + Fencing, + NoLeader, + NodeUnavailable, + Capacity, + BalanceUnavailable, + BalanceExhausted, + TargetUnavailable, + PullFailed, + LaunchFailed, + ReadinessFailed, + ExitFailure, + OutOfMemory, + NodePressure, + LeaseExpired, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CurrentInstance { + #[serde(with = "decimal_u64")] + pub generation: u64, + pub state: ObservedState, + pub observed_revision: Option, + /// Application snapshot identity only. No values or hashes of values. + pub applied_env_generation: Option, + pub exit_code: Option, + pub oom_killed: bool, + pub condition: Condition, + /// Last accepted cumulative measurement for this exact generation. Missing + /// or not-yet-reported usage is None, never manufactured zero consumption. + pub usage: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationalState { + pub desired_revision: Hash, + pub desired_state: DesiredState, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub condition: Condition, + pub restart_pending: bool, + pub restart_attempt: u32, + /// Unix milliseconds, represented exactly for browser clients. + #[serde(with = "decimal_u64")] + pub restart_not_before_ms: u64, + /// Reports only this operational generation. A historical attempt is never + /// presented as the currently authorized replacement. + pub current_instance: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum EndpointStatus { + Available { endpoints: Vec }, + Pending, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerStatus { + pub database_identity: Identity, + pub published: bool, + /// Latest published image and limits, which may differ from a draining + /// instance. No argv, environment, credentials, or private node addresses. + pub configuration: Option, + pub operational: Option, + /// Address discovery is independent of readiness and remains available + /// while stopped. Pending never means an empty declaration. + pub endpoints: EndpointStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerErrorCode { + InvalidRequest, + AccessDenied, + NotFound, + Conflict, + Unavailable, + OutcomeUnknown, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerApiError { + pub error: ContainerErrorCode, +} + +pub(super) mod decimal_u64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn receipt_preserves_full_generation_and_rejects_lossy_or_extra_fields() { + let receipt = ContainerOperationReceipt { + database_identity: Identity::ONE, + request_id: Uuid::from_u128(0x01950000000070008000000000000001), + action: ContainerAction::Restart, + generation: u64::MAX, + }; + let mut value = serde_json::to_value(&receipt).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + receipt + ); + for generation in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("18446744073709551616"), + ] { + value["generation"] = generation; + assert!(serde_json::from_value::(value.clone()).is_err()); + } + value = serde_json::to_value(receipt).unwrap(); + value["credential"] = serde_json::json!("unexpected"); + assert!(serde_json::from_value::(value).is_err()); + } + #[test] + fn pending_discovery_is_distinct_from_empty() { + assert_ne!( + serde_json::to_value(EndpointStatus::Pending).unwrap(), + serde_json::to_value(EndpointStatus::Available { endpoints: vec![] }).unwrap() + ); + } +} diff --git a/crates/lib/src/container/operations/resources.rs b/crates/lib/src/container/operations/resources.rs new file mode 100644 index 00000000000..8d45bdd0a48 --- /dev/null +++ b/crates/lib/src/container/operations/resources.rs @@ -0,0 +1,126 @@ +//! Browser-safe resource metadata and accepted cumulative usage, not live gauges. +use super::decimal_u64; +use crate::container::OciDigest; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfiguration { + pub image_digest: OciDigest, + pub resources: ResourceLimits, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceLimits { + #[serde(with = "decimal_u64")] + pub cpu_millicores: u64, + #[serde(with = "decimal_u64")] + pub memory_bytes: u64, + #[serde(with = "decimal_u64")] + pub scratch_bytes: u64, + #[serde(with = "decimal_u64")] + pub pids_max: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReportedUsage { + #[serde(with = "decimal_u64")] + pub sample_sequence: u64, + pub cumulative: UsageTotals, + /// Accounting has accepted its final totals. This flag does not establish + /// the container's observed state or physical storage reclamation. + pub final_report: bool, + /// A host reboot interrupted measurement in this or an earlier segment. + /// Cumulative totals include known usage only; the gap is not zero usage. + pub measurement_interrupted: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UsageTotals { + #[serde(with = "decimal_u128")] + pub cpu_nanoseconds: u128, + #[serde(with = "decimal_u128")] + pub memory_byte_seconds: u128, + #[serde(with = "decimal_u128")] + pub scratch_byte_seconds: u128, + #[serde(with = "decimal_u128")] + pub transmitted_bytes: u128, + #[serde(with = "decimal_u128")] + pub received_bytes: u128, +} + +mod decimal_u128 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u128, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u128 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u128")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cumulative_usage_preserves_large_counters_and_measurement_gaps() { + let usage = ReportedUsage { + sample_sequence: u64::MAX, + cumulative: UsageTotals { + cpu_nanoseconds: u128::MAX, + memory_byte_seconds: u128::MAX - 1, + scratch_byte_seconds: 0, + transmitted_bytes: 9_007_199_254_740_993, + received_bytes: 7, + }, + final_report: true, + measurement_interrupted: true, + }; + let encoded = serde_json::to_value(&usage).unwrap(); + assert_eq!(encoded["cumulative"]["cpu_nanoseconds"], u128::MAX.to_string()); + assert_eq!(encoded["cumulative"]["transmitted_bytes"], "9007199254740993"); + assert_eq!(serde_json::from_value::(encoded.clone()).unwrap(), usage); + for value in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("-1"), + serde_json::json!("340282366920938463463374607431768211456"), + ] { + let mut invalid = encoded.clone(); + invalid["cumulative"]["cpu_nanoseconds"] = value; + assert!(serde_json::from_value::(invalid).is_err()); + } + let mut invalid = encoded; + invalid["credential"] = serde_json::json!("must not appear"); + assert!(serde_json::from_value::(invalid).is_err()); + } + + #[test] + fn configuration_uses_the_oci_digest_and_exact_limits() { + let configuration = ContainerConfiguration { + image_digest: format!("sha256:{}", "ab".repeat(32)).parse().unwrap(), + resources: ResourceLimits { + cpu_millicores: 500, + memory_bytes: u64::MAX, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + }; + let encoded = serde_json::to_value(&configuration).unwrap(); + assert_eq!(encoded["image_digest"], format!("sha256:{}", "ab".repeat(32))); + assert_eq!(encoded["resources"]["memory_bytes"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + configuration + ); + } +} diff --git a/crates/lib/src/container/tests.rs b/crates/lib/src/container/tests.rs new file mode 100644 index 00000000000..dd7c208f43c --- /dev/null +++ b/crates/lib/src/container/tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn spec() -> ContainerSpec { + ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec!["API_KEY".into(), "DATABASE_URL".into()], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![ContainerPort { + name: "http".into(), + port: 8080, + protocol: PortProtocol::Http, + exposure: PortExposure::Public, + readiness_probe: ReadinessProbe::default(), + }], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } +} + +#[test] +fn oci_digest_is_strict_and_cannot_be_used_for_path_traversal() { + let digest = format!("sha256:{}", "ab".repeat(32)); + assert_eq!(digest.parse::().unwrap().to_string(), digest); + for value in [ + "sha256:../../host", + "sha256:abc", + "sha512:abc", + "blake3:abc", + "SHA256:abc", + "sha256:", + ] { + assert!(value.parse::().is_err(), "{value}"); + } + assert!(format!("sha256:{}", "AB".repeat(32)).parse::().is_err()); + assert!(format!("sha256:{}\n", "ab".repeat(32)).parse::().is_err()); +} + +#[test] +fn canonical_hash_ignores_declaration_order_but_preserves_launch_semantics() { + let limits = ContainerSpecLimits::default(); + let mut original = spec(); + original.ports.push(ContainerPort { + name: "admin".into(), + port: 8081, + ..original.ports[0].clone() + }); + let hash = original.canonical_hash(&limits).unwrap(); + let mut reordered = original.clone(); + reordered.env_keys.reverse(); + reordered.ports.reverse(); + assert_eq!(reordered.canonical_hash(&limits).unwrap(), hash); + let mut changed = reordered; + changed.argv.push("--read-only".into()); + assert_ne!(changed.canonical_hash(&limits).unwrap(), hash); + assert_ne!( + ContainerSpec { + resources: ContainerResources { + pids_max: 63, + ..original.resources + }, + ..original.clone() + } + .canonical_hash(&limits) + .unwrap(), + hash + ); + let bytes = bsatn::to_vec(&original).unwrap(); + assert_eq!(bsatn::from_slice::(&bytes).unwrap(), original); +} + +#[test] +fn duplicate_declarations_are_not_silently_deduplicated() { + let limits = ContainerSpecLimits::default(); + let mut value = spec(); + value.env_keys.push(value.env_keys[0].clone()); + assert_eq!(value.normalize(&limits).unwrap_err().field, "env_keys"); + let mut value = spec(); + value.ports.push(ContainerPort { + name: "another".into(), + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); + let mut value = spec(); + value.ports.push(ContainerPort { + port: 9090, + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); +} + +#[test] +fn readiness_cannot_change_authority_or_inject_a_request() { + for path in [ + "https://internal/", + "//internal/", + "/\\internal/", + "/health\r\nHost: internal", + "/health#fragment", + ] { + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: path.into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "readiness_probe.path" + ); + } + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: "/health?full=1".into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + value.validate(&ContainerSpecLimits::default()).unwrap(); +} + +#[test] +fn stage_one_rejects_mounts_and_unsupported_platforms() { + let mut value = spec(); + value.mounts.push(ContainerMount { + database: "self".into(), + source: "/".into(), + target: "/spacetime".into(), + read_only: false, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "mounts" + ); + let mut value = spec(); + value.image_platform.os = "windows".into(); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "image_platform" + ); +} + +#[test] +fn startup_size_counts_environment_and_pointers_without_leaking_values() { + let secret = "THIS_VALUE_MUST_NOT_APPEAR_IN_ERRORS"; + let env = vec![format!("KEY={secret}\0")]; + let error = validate_exec_size(&spec().argv, &env).unwrap_err().to_string(); + assert!(!error.contains(secret)); + assert!(!error.contains("KEY=")); + let bounded_strings = vec!["x".repeat(MAX_EXEC_STRING_BYTES - 1); 5]; + assert!(validate_exec_size(&spec().argv, &bounded_strings).is_err()); + let many_empty_entries = vec![String::new(); MAX_EXEC_BYTES / 8]; + assert!(validate_exec_size(&[], &many_empty_entries).is_err()); + assert!(validate_exec_size(&spec().argv, &["KEY=value".into()]).is_ok()); +} + +#[test] +fn tenant_environment_cannot_override_platform_discovery_or_credentials() { + for key in [ + "SPACETIMEDB_DATABASE_IDENTITY", + "SPACETIMEDB_SERVER_URI", + "SPACETIMEDB_CREDENTIAL_BROKER", + "SPACETIMEDB_TOKEN", + "A=B", + "0BAD", + "BAD\0KEY", + ] { + assert!(validate_env_key(key).is_err(), "{key:?}"); + } + for key in ["API_KEY", "_CUSTOM", "path", "A1"] { + validate_env_key(key).unwrap(); + } +} + +#[test] +fn zero_or_overflowing_resource_requests_are_not_admitted() { + let limits = ContainerSpecLimits::default(); + for cpu in [0, u64::MAX, limits.resources.cpu_millicores + 1] { + let mut value = spec(); + value.resources.cpu_millicores = cpu; + assert_eq!(value.validate(&limits).unwrap_err().field, "resources"); + } + let mut value = spec(); + value.resources.scratch_bytes = 0; + assert!(value.validate(&limits).is_err()); +} + +#[test] +fn successful_jobs_and_on_failure_services_remain_terminal() { + assert!(!RestartPolicy::Never.restarts_after(None)); + assert!(!RestartPolicy::Never.restarts_after(Some(1))); + assert!(!RestartPolicy::OnFailure.restarts_after(Some(0))); + assert!(RestartPolicy::OnFailure.restarts_after(None)); + assert!(RestartPolicy::OnFailure.restarts_after(Some(1))); + let mut job = spec(); + job.mode = ContainerMode::Job; + job.restart = RestartPolicy::Always; + assert_eq!( + job.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "restart" + ); +} + +#[cfg(feature = "serde")] +#[test] +fn json_requires_explicit_port_exposure_and_rejects_privileged_fields() { + let original = spec(); + let json = serde_json::to_value(&original).unwrap(); + assert_eq!(serde_json::from_value::(json.clone()).unwrap(), original); + let mut missing = json.clone(); + missing["ports"][0].as_object_mut().unwrap().remove("exposure"); + assert!(serde_json::from_value::(missing).is_err()); + let mut injected = json; + injected + .as_object_mut() + .unwrap() + .insert("privileged".into(), true.into()); + assert!(serde_json::from_value::(injected).is_err()); + let keep: ContainerAction = serde_json::from_str(r#"{"action":"keep"}"#).unwrap(); + assert_eq!(keep, ContainerAction::Keep); + let remove: ContainerAction = serde_json::from_str(r#"{"action":"remove"}"#).unwrap(); + assert_eq!(remove, ContainerAction::Remove); + assert!(serde_json::from_str::(r#"{"action":"remove","value":{}}"#).is_err()); +} + +#[test] +fn concrete_container_actions_preserve_wire_tags() { + let container = spec(); + assert_eq!(bsatn::to_vec(&ContainerAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ContainerAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&container).unwrap()); + assert_eq!(bsatn::to_vec(&ContainerAction::Set(container)).unwrap(), expected); +} diff --git a/crates/lib/src/container_environment.rs b/crates/lib/src/container_environment.rs new file mode 100644 index 00000000000..e33073c2cba --- /dev/null +++ b/crates/lib/src/container_environment.rs @@ -0,0 +1,43 @@ +//! Immutable application environment identity, independent of expiring launch credentials. +//! +//! These values describe a request. They do not authenticate a client or grant +//! access to environment values. Only the trusted host/control protocol may +//! capture, read, or close a snapshot. + +use crate::{Hash, Identity, SpacetimeType, Uuid}; + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotScope { + pub cluster: String, + pub database_id: u64, + pub database_identity: Identity, + pub node_id: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub node_incarnation: Uuid, + pub generation: u64, + pub deployment_revision: Hash, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub publication_operation: Uuid, + pub publication_epoch: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub start_request: Uuid, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub env_generation: Uuid, + /// The complete sorted, unique key list of the committed container declaration. + pub env_keys: Vec, +} + +/// Stable identity of a committed capture. No secret values or value hashes. +/// A durability barrier belongs to each proof, not to this immutable receipt. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotReceipt { + pub scope: EnvironmentSnapshotScope, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub capture_receipt: Uuid, +} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs new file mode 100644 index 00000000000..d5e77ad28d0 --- /dev/null +++ b/crates/lib/src/deployment.rs @@ -0,0 +1,272 @@ +//! Normalized deployment protocol shared by publication coordinators and hosts. +//! +//! Module artifacts and OCI objects are uploaded separately. A deployment is +//! immutable configuration, never a place for runtime credentials or env values. + +use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; +use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; + +#[cfg(feature = "serde")] +pub mod api; +pub mod manifest; +pub mod system_empty; +pub use system_empty::SystemEmptyModule; + +pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; +pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = system_empty::VERSION; +pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; +pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; +pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum UserModuleKind { + Wasm, + Js, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct UserModule { + pub kind: UserModuleKind, + /// The existing module program hash, not an OCI object digest. + pub program_hash: Hash, +} + +/// Explicit module replacement/removal, with its own exported schema name. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleAction { + #[default] + Keep, + Set(UserModule), + /// Remove user code and select the exact platform module generated from + /// container configuration. Schema changes use this explicit action too. + Remove(SystemEmptyModule), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleComponent { + SystemEmpty(SystemEmptyModule), + User(UserModule), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct DeploymentSpecV1 { + pub module: ModuleComponent, + pub container: Option, +} + +/// Persist the discriminant along with the payload. Unknown encodings fail to +/// decode; a restore must never fall back to an empty or older deployment. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "deployment", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum DeploymentSpec { + V1(DeploymentSpecV1), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PublishEnvelope { + pub version: u32, + #[cfg_attr(feature = "serde", serde(with = "uuid_json"))] + pub operation_id: Uuid, + /// Exact compare-and-set precondition. None means no deployment record has + /// been installed yet, rather than permission to overwrite any revision. + pub expected_revision: Option, + /// Last committed operation paired with the revision; ENV-only publications + /// can retain the same revision while changing this cursor. + #[cfg_attr(feature = "serde", serde(with = "option_uuid_json"))] + pub expected_last_operation: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub module_action: ModuleAction, + #[cfg_attr(feature = "serde", serde(default))] + pub container_action: ContainerAction, +} + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentValidationError { + #[error("unsupported publish protocol version")] + UnsupportedVersion, + #[error("unsupported platform empty-module version")] + UnsupportedEmptyModule, + #[error("operation_id must be a version 7 UUID")] + InvalidOperationId, + #[error("publication operation has expired")] + ExpiredOperation, + #[error("publication operation timestamp is too far in the future")] + FutureOperation, + #[error("deployment exceeds the protocol size limit")] + TooLarge, + #[error("deployment encoding is invalid or unsupported")] + InvalidEncoding, + #[error(transparent)] + Container(#[from] ContainerValidationError), +} + +impl DeploymentSpec { + pub fn current(&self) -> &DeploymentSpecV1 { + match self { + Self::V1(spec) => spec, + } + } + + pub fn normalize(self, limits: &ContainerSpecLimits) -> Result { + let Self::V1(mut spec) = self; + if let ModuleComponent::SystemEmpty(module) = &spec.module + && module.version != SYSTEM_EMPTY_MODULE_VERSION + { + return Err(DeploymentValidationError::UnsupportedEmptyModule); + } + spec.container = spec.container.map(|spec| spec.normalize(limits)).transpose()?; + let spec = Self::V1(spec); + spec.encode()?; + Ok(spec) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding) + } + + /// Call on the result of normalize. Unlike a request fingerprint, this is + /// independent of operation ID, publisher, and mutable execution status. + pub fn revision(&self) -> Result { + let mut bytes = b"spacetimedb/deployment\0".to_vec(); + bytes.extend_from_slice(&self.encode()?); + Ok(hash_bytes(bytes)) + } +} + +impl PublishEnvelope { + pub fn resolve( + &self, + previous: Option<&DeploymentSpec>, + limits: &ContainerSpecLimits, + ) -> Result { + if self.version != PUBLISH_PROTOCOL_VERSION { + return Err(DeploymentValidationError::UnsupportedVersion); + } + use spacetimedb_sats::uuid::Version; + if !matches!(self.operation_id.get_version(), Some(Version::V7)) { + return Err(DeploymentValidationError::InvalidOperationId); + } + if self.expected_revision.is_some() != self.expected_last_operation.is_some() { + return Err(DeploymentValidationError::InvalidEncoding); + } + if self + .expected_last_operation + .is_some_and(|id| id.get_version() != Some(Version::V7)) + { + return Err(DeploymentValidationError::InvalidOperationId); + } + let prior = previous.map(DeploymentSpec::current); + let module = match &self.module_action { + ModuleAction::Keep => prior + .map(|p| p.module.clone()) + .unwrap_or(ModuleComponent::SystemEmpty(system_empty::empty().descriptor)), + ModuleAction::Set(module) => ModuleComponent::User(module.clone()), + ModuleAction::Remove(module) => ModuleComponent::SystemEmpty(*module), + }; + let container = match &self.container_action { + ContainerAction::Keep => prior.and_then(|p| p.container.clone()), + ContainerAction::Set(container) => Some(container.clone()), + ContainerAction::Remove => None, + }; + DeploymentSpec::V1(DeploymentSpecV1 { module, container }).normalize(limits) + } + + pub fn requires_container_permission(&self) -> bool { + matches!(self.container_action, ContainerAction::Set(_)) + } +} + +/// UUIDv7 embeds its creation millisecond. This makes an expired retry +/// distinguishable from a new request even after its ledger row is collected. +/// A fresh UUID with a changed timestamp is a different operation. +pub fn operation_expiry_ms(id: Uuid, now_ms: u64) -> Result { + if id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let created_ms = (id.as_u128() >> 80) as u64; + if created_ms > now_ms.saturating_add(MAX_OPERATION_CLOCK_SKEW_MS) { + return Err(DeploymentValidationError::FutureOperation); + } + let expires_ms = created_ms + PUBLISH_RETRY_WINDOW_MS; + if now_ms >= expires_ms { + return Err(DeploymentValidationError::ExpiredOperation); + } + Ok(expires_ms) +} + +#[cfg(feature = "serde")] +pub mod uuid_json { + use super::Uuid; + pub fn serialize(id: &Uuid, serializer: S) -> Result { + serializer.collect_str(id) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Uuid::parse_str(&value).map_err(serde::de::Error::custom) + } +} + +#[cfg(feature = "serde")] +pub mod option_uuid_json { + use super::Uuid; + pub fn serialize(id: &Option, serializer: S) -> Result { + match id { + Some(id) => serializer.serialize_some(&id.to_string()), + None => serializer.serialize_none(), + } + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let value = as serde::Deserialize>::deserialize(deserializer)?; + value + .map(|value| Uuid::parse_str(&value).map_err(serde::de::Error::custom)) + .transpose() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/api.rs b/crates/lib/src/deployment/api.rs new file mode 100644 index 00000000000..f69054661bd --- /dev/null +++ b/crates/lib/src/deployment/api.rs @@ -0,0 +1,143 @@ +//! HTTP publication messages shared by the CLI, dashboard and Cloud. Artifacts +//! are uploaded separately. Complete environment values belong only to the +//! protected publication request; public status and manifests remain value-free. + +use super::{ + manifest::PreparedDeploymentManifest, option_uuid_json, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION, +}; +use crate::{container::OciDigest, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +mod request_decode; +pub use request_decode::{PublishRequestError, MAX_PUBLISH_REQUEST_BYTES}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReference { + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + pub manifest: PreparedDeploymentManifest, + /// Must match the server-generated reservation when creating a database. + pub creation: Option, + /// Original uploaded OCI index or executable manifest. Required for Set. + /// Keep uses the prior retained executable manifest; Remove has no image. + pub image_source: Option, + /// Complete private values. Omission means an empty replacement, never Keep. + #[serde(default, deserialize_with = "request_decode::deserialize_environment")] + pub environment: BTreeMap, +} + +impl std::fmt::Debug for PublishRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishRequest") + .field("manifest", &self.manifest) + .field("creation", &self.creation) + .field("image_source", &self.image_source) + .field("environment", &"[redacted]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreationOptions { + pub parent: Option, + pub organization: Option, + pub num_replicas: Option, + #[serde(default = "default_anti_affinity")] + pub enforce_anti_affinity: bool, +} +fn default_anti_affinity() -> bool { + true +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReserveDatabaseRequest { + pub version: u32, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub options: CreationOptions, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DatabaseReservation { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub expires_at: String, + pub staging_open: bool, + pub artifact_endpoint: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicationPhase { + Prepared, + Quiescing, + Committed, + Activating, + Complete, + AbortedBeforeCommit, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationStatus { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub phase: PublicationPhase, + pub expected_revision: Option, + #[serde(with = "option_uuid_json")] + pub expected_last_operation: Option, + pub publication_epoch: u64, + pub proposed_revision: Hash, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeploymentStatus { + pub database_identity: Identity, + /// None means this database has not yet used managed publication. + pub revision: Option, + #[serde(with = "option_uuid_json")] + pub last_operation: Option, + pub deployment: DeploymentSpec, + /// Lets Keep select exactly the currently installed program bytes. + pub module_artifact: ArtifactReference, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishPermission { + pub identity: Identity, + pub can_publish: bool, + /// Decimal string preserves all u64 revision values in browser clients. + pub source_revision: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationCapabilities { + pub version: u32, + pub enabled: bool, + pub artifact_endpoint: Option, +} +impl PublicationCapabilities { + pub fn disabled() -> Self { + Self { + version: PUBLISH_PROTOCOL_VERSION, + enabled: false, + artifact_endpoint: None, + } + } +} diff --git a/crates/lib/src/deployment/api/request_decode.rs b/crates/lib/src/deployment/api/request_decode.rs new file mode 100644 index 00000000000..9bb6f947841 --- /dev/null +++ b/crates/lib/src/deployment/api/request_decode.rs @@ -0,0 +1,86 @@ +use super::PublishRequest; +use crate::deployment::MAX_DEPLOYMENT_BYTES; +use crate::environment::{validate_key, validate_value, MAX_ENV_KEY_BYTES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS}; +use serde::de::{MapAccess, Visitor}; +use std::{collections::BTreeMap, fmt}; + +/// Conservative JSON escaping allowance, independent of decoded metadata/map +/// limits. Checked constant arithmetic fails compilation if limits overflow. +pub const MAX_PUBLISH_REQUEST_BYTES: usize = { + let metadata = MAX_DEPLOYMENT_BYTES.checked_mul(6).expect("metadata JSON bound"); + let entry = MAX_ENV_KEY_BYTES + .checked_add(MAX_ENV_VALUE_BYTES) + .expect("environment entry bytes") + .checked_mul(6) + .expect("environment JSON escaping") + .checked_add(8) + .expect("environment JSON framing"); + metadata + .checked_add(MAX_ENV_VARS.checked_mul(entry).expect("environment JSON bound")) + .expect("publication JSON sections") + .checked_add(4096) + .expect("publication JSON framing") +}; + +/// Never retains a value-bearing parser error, source error, input or digest. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid publication request")] +pub struct PublishRequestError; + +impl PublishRequest { + /// Decode protected HTTP/journal input. Routes must also bound streamed + /// bytes and concurrent body ownership before buffering reaches this method. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_PUBLISH_REQUEST_BYTES { + return Err(PublishRequestError); + } + let request: Self = serde_json::from_slice(bytes).map_err(|_| PublishRequestError)?; + request.validate_structure()?; + Ok(request) + } + + /// Structural limits also apply to callers constructing a typed request. + /// Configured resource policy and selected-module declarations are checked + /// at publication admission, not inferred from environment key selection. + pub fn validate_structure(&self) -> Result<(), PublishRequestError> { + self.manifest.encode().map_err(|_| PublishRequestError)?; + if self.environment.len() > MAX_ENV_VARS { + return Err(PublishRequestError); + } + for (key, value) in &self.environment { + validate_key(key).map_err(|_| PublishRequestError)?; + validate_value(value).map_err(|_| PublishRequestError)?; + } + Ok(()) + } +} + +pub(super) fn deserialize_environment<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct EnvironmentVisitor; + impl<'de> Visitor<'de> for EnvironmentVisitor { + type Value = BTreeMap; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a bounded publication environment object") + } + fn visit_map>(self, mut input: M) -> Result { + let mut values = BTreeMap::new(); + while let Some(key) = input.next_key::()? { + if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { + return Err(serde::de::Error::custom("invalid publication environment")); + } + let value = input.next_value::()?; + if validate_value(&value).is_err() { + return Err(serde::de::Error::custom("invalid publication environment")); + } + values.insert(key, value); + } + Ok(values) + } + } + deserializer.deserialize_map(EnvironmentVisitor) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/api/request_decode/tests.rs b/crates/lib/src/deployment/api/request_decode/tests.rs new file mode 100644 index 00000000000..16b4c8b2e5d --- /dev/null +++ b/crates/lib/src/deployment/api/request_decode/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use crate::container::{ContainerAction, OciDigest}; +use crate::deployment::manifest::{ + ModuleArtifact, PreparedDeploymentManifest, PreparedDeploymentManifestV1, PreparedMigrationPolicy, +}; +use crate::deployment::{ + DeploymentSpec, DeploymentSpecV1, ModuleAction, ModuleComponent, PublishEnvelope, PUBLISH_PROTOCOL_VERSION, +}; +use crate::Uuid; + +fn request() -> PublishRequest { + PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap(), + expected_revision: None, + expected_last_operation: None, + module_action: ModuleAction::Keep, + container_action: ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(crate::deployment::system_empty::empty().descriptor), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: None, + image_source: None, + environment: BTreeMap::new(), + } +} + +fn with_raw_environment(raw: &str) -> Vec { + let mut value = serde_json::to_value(request()).unwrap(); + value.as_object_mut().unwrap().remove("environment"); + let mut json = serde_json::to_string(&value).unwrap(); + json.pop(); + format!("{json},\"environment\":{raw}}}").into_bytes() +} + +#[test] +fn protected_environment_decode_is_complete_redacted_and_preserves_strings() { + let mut original = request(); + original.environment = BTreeMap::from([ + ("TOKEN".into(), "private-secret-sentinel".into()), + ("EMPTY".into(), String::new()), + ("NUL".into(), "a\0b".into()), + ("UTF8".into(), "日本語".into()), + ]); + let bytes = serde_json::to_vec(&original).unwrap(); + assert_eq!(PublishRequest::decode(&bytes).unwrap(), original); + let debug = format!("{original:?}"); + assert!(debug.contains("[redacted]")); + for (key, value) in &original.environment { + assert!(!debug.contains(key)); + if !value.is_empty() { + assert!(!debug.contains(value)); + } + } + let mut without = serde_json::to_value(original).unwrap(); + without.as_object_mut().unwrap().remove("environment"); + assert!(PublishRequest::decode(&serde_json::to_vec(&without).unwrap()) + .unwrap() + .environment + .is_empty()); + assert!(PublishRequest::decode(&with_raw_environment("{}")) + .unwrap() + .environment + .is_empty()); +} + +#[test] +fn duplicate_malformed_and_invalid_environment_errors_never_echo_input() { + for raw in [ + r#"{"TOKEN":"private-secret-sentinel","TOKEN":"other"}"#, + r#"{"TOKEN":"private-secret-sentinel","\u0054OKEN":"other"}"#, + r#"{"TOKEN":{"private-secret-sentinel":true}}"#, + r#"{"private-secret-sentinel":"value"}"#, + r#"{"TOKEN":123}"#, + r#"{"TOKEN":"private-secret-sentinel""#, + "null", + "[]", + ] { + let error = PublishRequest::decode(&with_raw_environment(raw)).unwrap_err(); + assert_eq!(error.to_string(), "invalid publication request"); + assert_eq!(format!("{error:?}"), "PublishRequestError"); + } +} + +#[test] +fn environment_and_raw_byte_limits_are_independent_of_manifest_size() { + let mut original = request(); + for index in 0..MAX_ENV_VARS { + let key = format!("K{index:03}{}", "K".repeat(MAX_ENV_KEY_BYTES - 4)); + original.environment.insert(key, "\0".repeat(MAX_ENV_VALUE_BYTES)); + } + let bytes = serde_json::to_vec(&original).unwrap(); + assert!(bytes.len() <= MAX_PUBLISH_REQUEST_BYTES); + assert_eq!(PublishRequest::decode(&bytes).unwrap(), original); + original.environment.insert("EXTRA".into(), String::new()); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); + original.environment.clear(); + for (key, value) in [ + ("K".repeat(MAX_ENV_KEY_BYTES + 1), String::new()), + ("K".into(), "é".repeat(MAX_ENV_VALUE_BYTES / 2 + 1)), + ] { + original.environment = BTreeMap::from([(key, value)]); + assert!(original.validate_structure().is_err()); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); + } + assert_eq!( + PublishRequest::decode(&vec![b' '; MAX_PUBLISH_REQUEST_BYTES + 1]), + Err(PublishRequestError) + ); + original.environment.clear(); + let PreparedDeploymentManifest::V1(manifest) = &mut original.manifest; + let DeploymentSpec::V1(deployment) = &mut manifest.deployment; + // Metadata retains its own byte cap even with an empty environment. + use crate::container::*; + let container = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["x".repeat(MAX_DEPLOYMENT_BYTES + 1)], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + }; + deployment.container = Some(container); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); +} + +#[test] +fn optional_publication_uuid_is_a_string_or_null_and_not_an_integer() { + let mut original = request(); + let PreparedDeploymentManifest::V1(manifest) = &mut original.manifest; + manifest.envelope.expected_last_operation = Some(manifest.envelope.operation_id); + let value = serde_json::to_value(&original).unwrap(); + let cursor = &value["manifest"]["manifest"]["envelope"]["expected_last_operation"]; + assert_eq!(cursor.as_str(), Some("01991ec4-0000-7000-8000-000000000001")); + assert_eq!( + PublishRequest::decode(&serde_json::to_vec(&value).unwrap()).unwrap(), + original + ); + let mut value = value; + value["manifest"]["manifest"]["envelope"]["expected_last_operation"] = 17.into(); + assert!(PublishRequest::decode(&serde_json::to_vec(&value).unwrap()).is_err()); +} diff --git a/crates/lib/src/deployment/manifest.rs b/crates/lib/src/deployment/manifest.rs new file mode 100644 index 00000000000..683599fd9ab --- /dev/null +++ b/crates/lib/src/deployment/manifest.rs @@ -0,0 +1,173 @@ +//! Immutable publication recovery inputs. The manifest's SHA-256 artifact +//! digest binds migration intent as well as the effective deployment. Its +//! digest differs from the deployment revision, which excludes migration policy. + +use super::{DeploymentSpec, DeploymentValidationError, PublishEnvelope, MAX_DEPLOYMENT_BYTES}; +use crate::container::{ContainerSpecLimits, OciDigest}; +use crate::{bsatn, Hash, SpacetimeType}; + +pub const MAX_MODULE_ARTIFACT_BYTES: u64 = 32 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ModuleArtifact { + /// SHA-256 of the complete stored bytes, distinct from the module's Keccak hash. + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "policy", content = "token", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum PreparedMigrationPolicy { + Compatible, + /// The existing migration token binds database Identity and old/new module + /// hashes. Recovery must retain the originally acknowledged policy. + BreakClients(Hash), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PreparedDeploymentManifestV1 { + pub envelope: PublishEnvelope, + pub deployment: DeploymentSpec, + /// Always retained, including for the bundled empty module. Genesis and + /// later recovery must select exactly the admitted program bytes. + pub module_artifact: ModuleArtifact, + pub migration_policy: PreparedMigrationPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "manifest", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum PreparedDeploymentManifest { + V1(PreparedDeploymentManifestV1), +} + +impl PreparedDeploymentManifest { + pub fn current(&self) -> &PreparedDeploymentManifestV1 { + match self { + Self::V1(manifest) => manifest, + } + } + + /// Validate before retaining the artifact. This checks the encoding and + /// metadata; the artifact service verifies SHA-256/length and the host + /// validates the selected program, capabilities and actual migration. + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), DeploymentValidationError> { + let manifest = self.current(); + if manifest.module_artifact.size_bytes == 0 || manifest.module_artifact.size_bytes > MAX_MODULE_ARTIFACT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + if manifest.deployment.clone().normalize(limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + // Using the prepared components as the Keep baseline verifies every + // explicit Set/Remove action without inventing a prior deployment. + // The host separately checks the real prior state under its fence. + if manifest.envelope.resolve(Some(&manifest.deployment), limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + self.encode()?; + Ok(()) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + let manifest: Self = bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + // Reject trailing or noncanonical bytes even if the decoder accepts + // them, so every retained descriptor names one unambiguous manifest. + if manifest.encode()?.as_ref() != bytes { + return Err(DeploymentValidationError::InvalidEncoding); + } + Ok(manifest) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::deployment::{DeploymentSpecV1, ModuleComponent}; + + fn manifest() -> PreparedDeploymentManifest { + PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: super::super::PUBLISH_PROTOCOL_VERSION, + operation_id: crate::Uuid::from_u128(0x01991ec4000070008000000000000001), + expected_revision: None, + expected_last_operation: None, + module_action: super::super::ModuleAction::Keep, + container_action: crate::container::ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(crate::deployment::system_empty::empty().descriptor), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }) + } + + #[test] + fn retained_manifest_preserves_migration_intent_without_changing_revision() { + let first = manifest(); + let mut acknowledged = first.clone(); + let PreparedDeploymentManifest::V1(value) = &mut acknowledged; + value.migration_policy = PreparedMigrationPolicy::BreakClients(Hash::from_byte_array([32; 32])); + assert_eq!( + first.current().deployment.revision().unwrap(), + acknowledged.current().deployment.revision().unwrap() + ); + let encoded = acknowledged.encode().unwrap(); + assert_ne!(first.encode().unwrap(), encoded); + assert_eq!(PreparedDeploymentManifest::decode(&encoded).unwrap(), acknowledged); + acknowledged.validate(&Default::default()).unwrap(); + } + + #[test] + fn retained_manifest_rejects_unknown_encoding_and_invalid_module_bounds() { + let mut value = manifest(); + let mut trailing = value.encode().unwrap().to_vec(); + trailing.push(0); + assert!(PreparedDeploymentManifest::decode(&trailing).is_err()); + let mut unknown = value.encode().unwrap().to_vec(); + unknown[0] = 1; + assert!(PreparedDeploymentManifest::decode(&unknown).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = 0; + assert!(value.validate(&Default::default()).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = MAX_MODULE_ARTIFACT_BYTES + 1; + assert!(value.validate(&Default::default()).is_err()); + } +} diff --git a/crates/lib/src/deployment/system_empty.rs b/crates/lib/src/deployment/system_empty.rs new file mode 100644 index 00000000000..cef92c5cb89 --- /dev/null +++ b/crates/lib/src/deployment/system_empty.rs @@ -0,0 +1,275 @@ +//! Canonical platform Wasm for a container database without a user module. +//! +//! The declared environment changes the program identity. Verification extracts +//! only bounded declaration data, then regenerates and compares every byte. It +//! never executes publisher-supplied code or trusts an empty-looking schema. + +use crate::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; +use crate::environment::{ + EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema, MAX_ENV_KEY_BYTES, MAX_ENV_SCHEMA_BYTES, + MAX_ENV_UNION_ENTRIES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS, +}; +use crate::{bsatn, hash_bytes, Hash, RawModuleDef, SpacetimeType}; + +pub const VERSION: u32 = 2; +const WASM_HEADER: &[u8] = b"\0asm\x01\0\0\0"; +// String bytes have their own aggregate limit. Include every possible BSATN +// length prefix, constraint tag, optional flag, and enclosing module section. +const MAX_METADATA_BYTES: usize = MAX_ENV_SCHEMA_BYTES + MAX_ENV_VARS * (10 + MAX_ENV_UNION_ENTRIES * 4) + 128; +pub const MAX_PROGRAM_BYTES: usize = MAX_METADATA_BYTES + 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct SystemEmptyModule { + pub version: u32, + pub program_hash: Hash, +} + +#[derive(Debug, thiserror::Error)] +#[error("invalid platform container module")] +pub struct InvalidSystemModule; + +type ModuleResult = std::result::Result; + +pub struct GeneratedModule { + pub descriptor: SystemEmptyModule, + pub bytes: Box<[u8]>, +} + +/// The canonical empty declaration schema used for an initial Keep or explicit +/// module removal without container environment declarations. +pub fn empty() -> &'static GeneratedModule { + static EMPTY: std::sync::OnceLock = std::sync::OnceLock::new(); + EMPTY.get_or_init(|| generate(&EnvironmentSchema::default()).expect("empty platform schema must encode")) +} + +/// Generate the exact versioned program for a previously validated schema. +pub fn generate(environment: &EnvironmentSchema) -> ModuleResult { + let metadata = RawModuleDef::V10(RawModuleDefV10 { + sections: vec![ + RawModuleDefV10Section::Typespace(Default::default()), + RawModuleDefV10Section::Environment(environment.declarations().cloned().collect()), + RawModuleDefV10Section::Capabilities(vec!["hosted_auth_v1".into()]), + ], + }); + let metadata = bsatn::to_vec(&metadata).map_err(|_| InvalidSystemModule)?; + if metadata.len() > MAX_METADATA_BYTES { + return Err(InvalidSystemModule); + } + // The size pointer is at16 and metadata starts at32. Fix both minimum and + // maximum memory to the smallest page count containing this exact schema. + let pages = (32 + metadata.len()).div_ceil(65536); + let mut wasm = WASM_HEADER.to_vec(); + section( + &mut wasm, + 1, + &[ + 4, 0x60, 3, 0x7f, 0x7f, 0x7f, 1, 0x7f, 0x60, 0, 1, 0x7f, 0x60, 1, 0x7f, 0, 0x60, 10, 0x7f, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, 0x7f, 1, 0x7f, + ], + ); + let mut imports = vec![2]; + name(&mut imports, b"spacetime_10.0"); + name(&mut imports, b"bytes_sink_write"); + imports.extend([0, 0]); + name(&mut imports, b"spacetime_10.7"); + name(&mut imports, b"get_call_auth_flags"); + imports.extend([0, 1]); + section(&mut wasm, 2, &imports); + section(&mut wasm, 3, &[2, 2, 3]); + let mut memory = vec![1, 1]; + leb(&mut memory, pages); + leb(&mut memory, pages); + section(&mut wasm, 5, &memory); + let mut exports = vec![3]; + name(&mut exports, b"memory"); + exports.extend([2, 0]); + name(&mut exports, b"__describe_module__"); + exports.extend([0, 2]); + name(&mut exports, b"__call_reducer__"); + exports.extend([0, 3]); + section(&mut wasm, 7, &exports); + // __describe_module__(sink): bytes_sink_write(sink,32,16), trap on error. + // __call_reducer__: unreachable, since there are no declared reducers. + section( + &mut wasm, + 10, + &[ + 2, 14, 0, 0x20, 0, 0x41, 0x20, 0x41, 0x10, 0x10, 0, 0x04, 0x40, 0, 0x0b, 0x0b, 3, 0, 0, 0x0b, + ], + ); + let mut data = vec![2, 0, 0x41, 0x10, 0x0b, 4]; + data.extend((metadata.len() as u32).to_le_bytes()); + data.extend([0, 0x41, 0x20, 0x0b]); + leb(&mut data, metadata.len()); + data.extend(metadata); + section(&mut wasm, 11, &data); + Ok(GeneratedModule { + descriptor: SystemEmptyModule { + version: VERSION, + program_hash: hash_bytes(&wasm), + }, + bytes: wasm.into(), + }) +} + +/// Verify the version, hash, declarations, executable code, exports, imports, +/// memory limits, and absence of additional sections without running the module. +pub fn verify(descriptor: &SystemEmptyModule, bytes: &[u8]) -> ModuleResult { + if descriptor.version != VERSION || bytes.len() > MAX_PROGRAM_BYTES { + return Err(InvalidSystemModule); + } + let environment = read_environment(bytes)?; + let expected = generate(&environment)?; + if expected.descriptor != *descriptor || expected.bytes.as_ref() != bytes { + return Err(InvalidSystemModule); + } + Ok(environment) +} + +fn read_environment(bytes: &[u8]) -> ModuleResult { + let mut wasm = Reader(bytes); + wasm.expect(WASM_HEADER)?; + let mut data = None; + while !wasm.0.is_empty() { + let tag = wasm.byte()?; + let len = wasm.leb()?; + let payload = wasm.take(len)?; + if tag == 11 && data.replace(payload).is_some() { + return Err(InvalidSystemModule); + } + } + let mut data = Reader(data.ok_or(InvalidSystemModule)?); + data.expect(&[2, 0, 0x41, 0x10, 0x0b, 4])?; + let size = data.u32()?; + if size > MAX_METADATA_BYTES { + return Err(InvalidSystemModule); + } + data.expect(&[0, 0x41, 0x20, 0x0b])?; + if data.leb()? != size { + return Err(InvalidSystemModule); + } + let mut metadata = Reader(data.take(size)?); + data.end()?; + // RawModuleDef::V10, exactly3 sections: empty Typespace, ENV15, Capabilities16. + metadata.expect(&[2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15])?; + let count = metadata.count(MAX_ENV_VARS)?; + let mut declarations = Vec::with_capacity(count); + let mut string_bytes = 0; + for _ in 0..count { + let name = metadata.string(MAX_ENV_KEY_BYTES, &mut string_bytes)?; + let constraint = match metadata.byte()? { + 0 => EnvironmentConstraint::AnyString, + 1 => EnvironmentConstraint::Literal(metadata.string(MAX_ENV_VALUE_BYTES, &mut string_bytes)?), + 2 => { + let count = metadata.count(MAX_ENV_UNION_ENTRIES)?; + let mut values = Vec::with_capacity(count); + for _ in 0..count { + values.push(metadata.string(MAX_ENV_VALUE_BYTES, &mut string_bytes)?); + } + EnvironmentConstraint::OneOf(values) + } + _ => return Err(InvalidSystemModule), + }; + let optional = match metadata.byte()? { + 0 => false, + 1 => true, + _ => return Err(InvalidSystemModule), + }; + declarations.push(EnvironmentDeclaration { + name, + constraint, + optional, + }); + } + metadata.expect(&[16, 1, 0, 0, 0, 14, 0, 0, 0])?; + metadata.expect(b"hosted_auth_v1")?; + metadata.end()?; + EnvironmentSchema::new(declarations).map_err(|_| InvalidSystemModule) +} + +fn leb(out: &mut Vec, mut value: usize) { + loop { + let byte = (value & 127) as u8; + value >>= 7; + out.push(byte | if value == 0 { 0 } else { 128 }); + if value == 0 { + break; + } + } +} + +fn name(out: &mut Vec, value: &[u8]) { + leb(out, value.len()); + out.extend(value); +} +fn section(out: &mut Vec, tag: u8, payload: &[u8]) { + out.push(tag); + name(out, payload); +} + +/// This reader accepts only the platform program's declaration framing, not +/// arbitrary Wasm or BSATN. Every length/count is bounded before allocation. +struct Reader<'a>(&'a [u8]); +impl<'a> Reader<'a> { + fn take(&mut self, len: usize) -> ModuleResult<&'a [u8]> { + let (head, tail) = self.0.split_at_checked(len).ok_or(InvalidSystemModule)?; + self.0 = tail; + Ok(head) + } + fn byte(&mut self) -> ModuleResult { + Ok(self.take(1)?[0]) + } + fn u32(&mut self) -> ModuleResult { + Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap()) as usize) + } + fn count(&mut self, limit: usize) -> ModuleResult { + let count = self.u32()?; + if count > limit { + return Err(InvalidSystemModule); + } + Ok(count) + } + fn leb(&mut self) -> ModuleResult { + let mut value = 0u32; + for shift in (0..35).step_by(7) { + let byte = self.byte()?; + if shift == 28 && byte > 15 { + return Err(InvalidSystemModule); + } + value |= u32::from(byte & 127) << shift; + if byte & 128 == 0 { + return Ok(value as usize); + } + } + Err(InvalidSystemModule) + } + fn string(&mut self, limit: usize, total: &mut usize) -> ModuleResult { + let len = self.count(limit)?; + *total += len; + if *total > MAX_ENV_SCHEMA_BYTES { + return Err(InvalidSystemModule); + } + let bytes = self.take(len)?; + Ok(std::str::from_utf8(bytes).map_err(|_| InvalidSystemModule)?.to_owned()) + } + fn expect(&mut self, bytes: &[u8]) -> ModuleResult<()> { + if self.take(bytes.len())? != bytes { + return Err(InvalidSystemModule); + } + Ok(()) + } + fn end(&self) -> ModuleResult<()> { + if self.0.is_empty() { + Ok(()) + } else { + Err(InvalidSystemModule) + } + } +} + +#[cfg(test)] +#[path = "system_empty/tests.rs"] +mod tests; diff --git a/crates/lib/src/deployment/system_empty/tests.rs b/crates/lib/src/deployment/system_empty/tests.rs new file mode 100644 index 00000000000..f5793a18eb1 --- /dev/null +++ b/crates/lib/src/deployment/system_empty/tests.rs @@ -0,0 +1,162 @@ +use super::*; + +fn declaration(name: &str, constraint: EnvironmentConstraint, optional: bool) -> EnvironmentDeclaration { + EnvironmentDeclaration { + name: name.into(), + constraint, + optional, + } +} + +#[test] +fn equivalent_normalized_schemas_select_the_same_program() { + let first = EnvironmentSchema::new(vec![ + declaration("TOKEN", EnvironmentConstraint::AnyString, true), + declaration( + "MODE", + EnvironmentConstraint::OneOf(vec!["blue".into(), "green".into()]), + false, + ), + ]) + .unwrap(); + let second = EnvironmentSchema::new(vec![ + declaration( + "MODE", + EnvironmentConstraint::OneOf(vec!["green".into(), "blue".into(), "blue".into()]), + false, + ), + declaration("TOKEN", EnvironmentConstraint::AnyString, true), + ]) + .unwrap(); + let generated = generate(&first).unwrap(); + let same = generate(&second).unwrap(); + assert_eq!(generated.bytes, same.bytes); + assert_eq!(generated.descriptor, same.descriptor); + assert_eq!(verify(&generated.descriptor, &generated.bytes).unwrap(), first); + let required = EnvironmentSchema::new(vec![declaration("TOKEN", EnvironmentConstraint::AnyString, false)]).unwrap(); + assert_ne!( + generated.descriptor.program_hash, + generate(&required).unwrap().descriptor.program_hash + ); +} + +#[test] +fn recognition_rejects_old_versions_forged_hashes_and_extra_executable_bytes() { + let generated = generate(&EnvironmentSchema::default()).unwrap(); + assert!(verify(&generated.descriptor, &generated.bytes).unwrap().is_empty()); + let mut descriptor = generated.descriptor; + descriptor.version = 1; + assert!(verify(&descriptor, &generated.bytes).is_err()); + descriptor = generated.descriptor; + descriptor.program_hash = Hash::ZERO; + assert!(verify(&descriptor, &generated.bytes).is_err()); + + let mut bytes = generated.bytes.to_vec(); + // Even a valid Wasm custom section with a correctly claimed new hash fails + // the exact platform-code check, rather than qualifying by schema alone. + bytes.extend([0, 2, 1, b'x']); + descriptor.program_hash = hash_bytes(&bytes); + assert!(verify(&descriptor, &bytes).is_err()); + for length in 0..generated.bytes.len() { + assert!(verify(&generated.descriptor, &generated.bytes[..length]).is_err()); + } +} + +#[test] +fn large_declarations_grow_fixed_memory_and_remain_bounded() { + let schema = EnvironmentSchema::new( + (0..255) + .map(|n| { + declaration( + &format!("K{n}"), + EnvironmentConstraint::Literal("x".repeat(MAX_ENV_VALUE_BYTES)), + false, + ) + }) + .collect(), + ) + .unwrap(); + let generated = generate(&schema).unwrap(); + assert!(generated.bytes.len() > 2_000_000); + assert!(generated.bytes.len() <= MAX_PROGRAM_BYTES); + assert_eq!(verify(&generated.descriptor, &generated.bytes).unwrap(), schema); + let mut wasm = Reader(&generated.bytes); + wasm.expect(WASM_HEADER).unwrap(); + let mut pages = None; + while !wasm.0.is_empty() { + let tag = wasm.byte().unwrap(); + let length = wasm.leb().unwrap(); + let mut payload = Reader(wasm.take(length).unwrap()); + if tag == 5 { + payload.expect(&[1, 1]).unwrap(); + let minimum = payload.leb().unwrap(); + let maximum = payload.leb().unwrap(); + assert_eq!(minimum, maximum); + payload.end().unwrap(); + pages = Some(minimum); + } + } + assert_eq!(pages, Some(32)); +} + +#[test] +fn hostile_counts_lengths_and_noncanonical_metadata_fail_before_allocation() { + let generated = generate(&EnvironmentSchema::default()).unwrap(); + let mut bytes = generated.bytes.to_vec(); + let prefix = [2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15]; + let offset = bytes.windows(prefix.len()).position(|window| window == prefix).unwrap() + prefix.len(); + bytes[offset..offset + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + let descriptor = SystemEmptyModule { + version: VERSION, + program_hash: hash_bytes(&bytes), + }; + assert!(verify(&descriptor, &bytes).is_err()); + + let mut bytes = WASM_HEADER.to_vec(); + bytes.extend([11, 0xff, 0xff, 0xff, 0xff, 0x0f]); + assert!(verify(&descriptor, &bytes).is_err()); + let mut bytes = WASM_HEADER.to_vec(); + bytes.extend([11, 0xff, 0xff, 0xff, 0xff, 0x10]); + assert!(verify(&descriptor, &bytes).is_err()); + let oversized = vec![0; MAX_PROGRAM_BYTES + 1]; + assert!(verify(&descriptor, &oversized).is_err()); +} + +#[test] +fn platform_imports_use_the_current_authorization_namespace() { + let generated = empty(); + let mut wasm = Reader(&generated.bytes); + wasm.expect(WASM_HEADER).unwrap(); + let mut imports = None; + while !wasm.0.is_empty() { + let tag = wasm.byte().unwrap(); + let length = wasm.leb().unwrap(); + let payload = wasm.take(length).unwrap(); + if tag == 2 { + assert!(imports.replace(payload).is_none()); + } + } + let mut expected = vec![2]; + name(&mut expected, b"spacetime_10.0"); + name(&mut expected, b"bytes_sink_write"); + expected.extend([0, 0]); + name(&mut expected, b"spacetime_10.7"); + name(&mut expected, b"get_call_auth_flags"); + expected.extend([0, 1]); + assert_eq!(imports, Some(expected.as_slice())); + + // A correctly hashed prototype namespace cannot identify the current + // platform module, even when declarations and descriptor version match. + let mut old = generated.bytes.to_vec(); + let namespace = b"spacetime_10.7"; + let offset = old + .windows(namespace.len()) + .position(|bytes| bytes == namespace) + .unwrap(); + old[offset + namespace.len() - 1] = b'6'; + let descriptor = SystemEmptyModule { + version: VERSION, + program_hash: hash_bytes(&old), + }; + assert!(verify(&descriptor, &old).is_err()); +} diff --git a/crates/lib/src/deployment/tests.rs b/crates/lib/src/deployment/tests.rs new file mode 100644 index 00000000000..2918d6c5dfe --- /dev/null +++ b/crates/lib/src/deployment/tests.rs @@ -0,0 +1,166 @@ +use super::*; + +fn operation_id() -> Uuid { + Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap() +} + +fn request(module_action: ModuleAction) -> PublishEnvelope { + PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: operation_id(), + expected_revision: None, + expected_last_operation: None, + module_action, + container_action: ContainerAction::Keep, + } +} + +#[test] +fn component_actions_preserve_or_explicitly_remove_the_module() { + let limits = ContainerSpecLimits::default(); + let user = UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(b"valid module artifact"), + }; + let set = request(ModuleAction::Set(user.clone())).resolve(None, &limits).unwrap(); + assert_eq!(set.current().module, ModuleComponent::User(user)); + let keep = request(ModuleAction::Keep).resolve(Some(&set), &limits).unwrap(); + assert_eq!(set, keep); + let remove = request(ModuleAction::Remove(system_empty::empty().descriptor)) + .resolve(Some(&set), &limits) + .unwrap(); + assert_eq!( + remove.current().module, + ModuleComponent::SystemEmpty(crate::deployment::system_empty::empty().descriptor) + ); + assert_ne!(remove.revision().unwrap(), keep.revision().unwrap()); +} + +#[test] +fn declared_platform_schema_changes_revision_and_keep_retains_its_program() { + use crate::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + let initial = request(ModuleAction::Keep).resolve(None, &Default::default()).unwrap(); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + let configured = system_empty::generate(&schema).unwrap(); + let selected = request(ModuleAction::Remove(configured.descriptor)) + .resolve(Some(&initial), &Default::default()) + .unwrap(); + assert_eq!( + selected.current().module, + ModuleComponent::SystemEmpty(configured.descriptor) + ); + assert_ne!(selected.revision().unwrap(), initial.revision().unwrap()); + assert_eq!( + request(ModuleAction::Keep) + .resolve(Some(&selected), &Default::default()) + .unwrap(), + selected + ); +} + +#[test] +fn concrete_module_actions_preserve_wire_tags_and_export_distinct_names() { + let module = UserModule { + kind: UserModuleKind::Js, + program_hash: hash_bytes(b"module"), + }; + assert_eq!(bsatn::to_vec(&ModuleAction::Keep).unwrap(), [0]); + let mut removed = vec![2]; + removed.extend(bsatn::to_vec(&system_empty::empty().descriptor).unwrap()); + assert_eq!( + bsatn::to_vec(&ModuleAction::Remove(system_empty::empty().descriptor)).unwrap(), + removed + ); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&module).unwrap()); + assert_eq!(bsatn::to_vec(&ModuleAction::Set(module)).unwrap(), expected); + + use crate::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_type::(); + let raw = builder.finish(); + let names: Vec<_> = raw + .sections + .iter() + .filter_map(|section| match section { + RawModuleDefV10Section::Types(types) => Some(types), + _ => None, + }) + .flatten() + .map(|ty| &*ty.source_name.source_name) + .collect(); + let distinct: std::collections::BTreeSet<_> = names.iter().copied().collect(); + assert_eq!( + names.len(), + distinct.len(), + "publish envelope exports duplicate type names" + ); + assert!(distinct.contains("ModuleAction")); + assert!(distinct.contains("ContainerAction")); +} + +#[test] +fn unknown_deployments_do_not_fall_back_to_an_empty_module() { + assert!(DeploymentSpec::decode(&[255]).is_err()); + let mut envelope = request(ModuleAction::Keep); + envelope.version += 1; + assert!(matches!( + envelope.resolve(None, &ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedVersion) + )); + let spec = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SystemEmptyModule { + version: 9000, + program_hash: Hash::ZERO, + }), + container: None, + }); + assert!(matches!( + spec.normalize(&ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedEmptyModule) + )); +} + +#[test] +fn operation_age_is_enforced_even_when_no_ledger_row_remains() { + let id = operation_id(); + let created_ms = (id.as_u128() >> 80) as u64; + assert_eq!( + operation_expiry_ms(id, created_ms).unwrap(), + created_ms + PUBLISH_RETRY_WINDOW_MS + ); + assert!(matches!( + operation_expiry_ms(id, created_ms + PUBLISH_RETRY_WINDOW_MS), + Err(DeploymentValidationError::ExpiredOperation) + )); + assert!(matches!( + operation_expiry_ms(id, created_ms - MAX_OPERATION_CLOCK_SKEW_MS - 1), + Err(DeploymentValidationError::FutureOperation) + )); + assert!(matches!( + operation_expiry_ms(Uuid::NIL, created_ms), + Err(DeploymentValidationError::InvalidOperationId) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn uuid_json_is_lossless_and_omission_means_keep() { + let envelope = request(ModuleAction::Keep); + let mut json = serde_json::to_value(&envelope).unwrap(); + assert_eq!(json["operation_id"], operation_id().to_string()); + json.as_object_mut().unwrap().remove("module_action"); + json.as_object_mut().unwrap().remove("container_action"); + let parsed: PublishEnvelope = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(parsed, envelope); + assert!(!parsed.requires_container_permission()); + json["publisher_identity"] = "untrusted owner override".into(); + assert!(serde_json::from_value::(json).is_err()); + let binary = bsatn::to_vec(&envelope).unwrap(); + assert_eq!(bsatn::from_slice::(&binary).unwrap(), envelope); +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 55d5621a16d..8918e7b90f7 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -11,7 +11,10 @@ use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; pub mod connection_id; +pub mod container; +pub mod container_environment; pub mod db; +pub mod deployment; mod direct_index_key; pub mod environment; pub mod error; diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml new file mode 100644 index 00000000000..f32bc83ab59 --- /dev/null +++ b/crates/oci/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spacetimedb-oci" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license-file = "../../LICENSE.txt" +description = "Bounded OCI image validation for SpacetimeDB container publishing" + +[dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10" +flate2.workspace = true +tar.workspace = true +zstd = "0.13" + +[lints] +workspace = true diff --git a/crates/oci/src/layers.rs b/crates/oci/src/layers.rs new file mode 100644 index 00000000000..c7d99cb39de --- /dev/null +++ b/crates/oci/src/layers.rs @@ -0,0 +1,373 @@ +//! Stream-verify layer expansion without extracting anything onto the host. +//! Resource admission uses measured bytes and bounded metadata, never compressed +//! sizes alone. The runtime must also enforce a dedicated finite cache filesystem. + +use crate::{Descriptor, OciDigest}; +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + io::{self, Read}, +}; + +#[derive(Clone, Copy, Debug)] +pub struct LayerLimits { + pub max_uncompressed_bytes: u64, + pub max_regular_file_bytes: u64, + pub max_entries: u64, + pub max_metadata_bytes: usize, + pub max_path_bytes: usize, + pub zstd_window_log_max: u32, +} +impl Default for LayerLimits { + fn default() -> Self { + Self { + max_uncompressed_bytes: 128 * 1024 * 1024 * 1024, + max_regular_file_bytes: 64 * 1024 * 1024 * 1024, + max_entries: 1_000_000, + max_metadata_bytes: 64 * 1024, + max_path_bytes: 4096, + zstd_window_log_max: 27, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VerifiedLayerSize { + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub regular_file_bytes: u64, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct VerifiedImageSize { + pub compressed_bytes: u64, + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub cache_reservation_bytes: u64, +} +impl VerifiedImageSize { + /// Conservative cache accounting, supplemented by the backend's hard + /// filesystem bound. Includes temporary compressed/unpacked copies and + /// per-entry/per-layer filesystem metadata. The trusted artifact service + /// supplies these verified measurements, never a publisher JSON field. + pub fn from_layers(compressed_bytes: u64, layers: &[VerifiedLayerSize]) -> Result { + let mut tar = 0u64; + let mut entries = 0u64; + for layer in layers { + tar = tar + .checked_add(layer.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(layer.entries) + .context("image entry count overflow")?; + } + let reservation = compressed_bytes + .checked_mul(2) + .and_then(|n| tar.checked_mul(3).and_then(|v| n.checked_add(v))) + .and_then(|n| entries.checked_mul(64 * 1024).and_then(|v| n.checked_add(v))) + .and_then(|n| { + (layers.len() as u64) + .checked_mul(16 * 1024 * 1024) + .and_then(|v| n.checked_add(v)) + }) + .context("image cache reservation overflow")?; + Ok(Self { + compressed_bytes, + uncompressed_tar_bytes: tar, + entries, + cache_reservation_bytes: reservation, + }) + } +} + +struct HashBounded { + inner: R, + hash: Sha256, + count: u64, + limit: u64, +} +impl Read for HashBounded { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + if self.count == self.limit { + let mut extra = [0u8; 1]; + if self.inner.read(&mut extra)? == 0 { + return Ok(0); + } + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "layer exceeds configured byte bound", + )); + } + let maximum = buf.len().min((self.limit - self.count).min(usize::MAX as u64) as usize); + let count = self.inner.read(&mut buf[..maximum])?; + self.count += count as u64; + self.hash.update(&buf[..count]); + Ok(count) + } +} + +pub fn verify_layer( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, +) -> Result { + verify_layer_with_check(reader, descriptor, diff_id, limits, || Ok(())) +} + +/// Check cancellation/deadline between both compressed and expanded reads. +/// The caller retains its worker admission until this synchronous operation ends. +pub fn verify_layer_with_check( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, + check: impl Fn() -> io::Result<()> + Copy, +) -> Result { + struct Checked { + reader: R, + check: F, + } + impl io::Result<()>> Read for Checked { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (self.check)()?; + self.reader.read(buf) + } + } + check()?; + ensure!( + descriptor.size > 0 && descriptor.size <= crate::MAX_IMAGE_BYTES, + "invalid compressed layer size" + ); + ensure!( + descriptor.urls.is_empty() && descriptor.data.is_none(), + "external layer sources are unsupported" + ); + let mut compressed = HashBounded { + inner: Checked { reader, check }, + hash: Sha256::new(), + count: 0, + limit: descriptor.size, + }; + let decoder: Box = match descriptor.media_type.as_str() { + "application/vnd.oci.image.layer.v1.tar" | "application/vnd.docker.image.rootfs.diff.tar" => { + Box::new(&mut compressed) + } + "application/vnd.oci.image.layer.v1.tar+gzip" | "application/vnd.docker.image.rootfs.diff.tar.gzip" => { + Box::new(flate2::read::MultiGzDecoder::new(&mut compressed)) + } + "application/vnd.oci.image.layer.v1.tar+zstd" => { + let mut decoder = zstd::stream::read::Decoder::new(&mut compressed)?; + decoder.window_log_max(limits.zstd_window_log_max)?; + Box::new(decoder) + } + _ => bail!("unsupported or foreign layer media type"), + }; + let mut expanded = HashBounded { + inner: Checked { reader: decoder, check }, + hash: Sha256::new(), + count: 0, + limit: limits.max_uncompressed_bytes, + }; + let (entries, regular_file_bytes) = scan_tar(&mut expanded, limits)?; + let uncompressed_tar_bytes = expanded.count; + ensure!( + OciDigest::sha256(expanded.hash.clone().finalize().into()) == diff_id, + "layer uncompressed SHA-256 does not match rootfs diff ID" + ); + drop(expanded); + ensure!( + compressed.count == descriptor.size + && OciDigest::sha256(compressed.hash.finalize().into()) == descriptor.digest, + "compressed layer SHA-256 or length mismatch" + ); + Ok(VerifiedLayerSize { + uncompressed_tar_bytes, + entries, + regular_file_bytes, + }) +} + +fn scan_tar(reader: &mut impl Read, limits: LayerLimits) -> Result<(u64, u64)> { + let mut entries = 0u64; + let mut regular_bytes = 0u64; + let mut pax = BTreeMap::new(); + let mut global = BTreeMap::new(); + let mut long_path = None; + let mut long_link = None; + loop { + let mut block = [0u8; 512]; + let first = reader.read(&mut block[..1])?; + if first == 0 { + break; + } + reader.read_exact(&mut block[1..]).context("truncated TAR header")?; + if block.iter().all(|&b| b == 0) { + continue; + } + entries = entries.checked_add(1).context("TAR entry count overflow")?; + ensure!(entries <= limits.max_entries, "too many layer entries"); + let header = tar::Header::from_byte_slice(&block); + let checksum = block[..148] + .iter() + .chain(&block[156..]) + .map(|&b| u32::from(b)) + .sum::() + + 8 * 32; + ensure!(header.cksum()? == checksum, "invalid TAR header checksum"); + let kind = header.entry_type().as_byte(); + let header_size = header.entry_size()?; + if matches!(kind, b'x' | b'g' | b'L' | b'K') { + ensure!( + header_size <= limits.max_metadata_bytes as u64, + "TAR metadata entry is too large" + ); + let mut bytes = vec![0; header_size as usize]; + reader.read_exact(&mut bytes)?; + skip_padding(reader, header_size)?; + match kind { + b'x' => { + ensure!(pax.is_empty(), "duplicate local PAX metadata"); + pax = parse_pax(&bytes)?; + } + b'g' => { + let update = parse_pax(&bytes)?; + global.extend(update); + ensure!( + global.len() <= 128 + && global.iter().map(|(k, v)| k.len() + v.len()).sum::() + <= limits.max_metadata_bytes, + "global PAX metadata exceeds bound" + ); + } + b'L' => { + ensure!(long_path.is_none(), "duplicate GNU long path"); + long_path = Some(trim_nul(bytes)); + } + b'K' => { + ensure!(long_link.is_none(), "duplicate GNU long link"); + long_link = Some(trim_nul(bytes)); + } + _ => unreachable!(), + } + continue; + } + let property = |name: &str| pax.get(name).or_else(|| global.get(name)); + let size = property("size") + .map(|v| v.parse::()) + .transpose()? + .unwrap_or(header_size); + ensure!(size <= limits.max_regular_file_bytes, "layer file exceeds size bound"); + let raw_path = header.path_bytes(); + let path = property("path") + .map(String::as_bytes) + .or(long_path.as_deref()) + .unwrap_or(&raw_path); + validate_path(path, limits.max_path_bytes, kind == b'5')?; + match kind { + 0 | b'0' | b'7' => { + regular_bytes = regular_bytes.checked_add(size).context("layer file size overflow")?; + } + b'5' => ensure!(size == 0, "directory entry has data"), + b'1' | b'2' => { + ensure!(size == 0, "link entry has data"); + let raw_link = header.link_name_bytes(); + let link = property("linkpath") + .map(String::as_bytes) + .or(long_link.as_deref()) + .or(raw_link.as_deref()) + .context("link target is missing")?; + ensure!( + !link.is_empty() && link.len() <= limits.max_path_bytes && !link.contains(&0), + "invalid link target" + ); + if kind == b'1' { + validate_path(link, limits.max_path_bytes, false)?; + } + // Absolute symbolic links are normal inside a Linux image. + // Extraction remains the confined runtime unpacker's job. + } + _ => bail!("unsupported sparse, special-device, or unknown TAR entry type"), + } + skip_exact(reader, size)?; + skip_padding(reader, size)?; + pax.clear(); + long_path = None; + long_link = None; + } + ensure!( + pax.is_empty() && long_path.is_none() && long_link.is_none(), + "orphaned TAR extension metadata" + ); + Ok((entries, regular_bytes)) +} + +fn trim_nul(mut bytes: Vec) -> Vec { + while bytes.last() == Some(&0) { + bytes.pop(); + } + bytes +} +fn skip_exact(reader: &mut impl Read, mut bytes: u64) -> Result<()> { + let mut buffer = [0u8; 64 * 1024]; + while bytes > 0 { + let count = bytes.min(buffer.len() as u64) as usize; + reader.read_exact(&mut buffer[..count])?; + bytes -= count as u64; + } + Ok(()) +} +fn skip_padding(reader: &mut impl Read, size: u64) -> Result<()> { + skip_exact(reader, (512 - size % 512) % 512) +} +fn validate_path(path: &[u8], max: usize, root_directory: bool) -> Result<()> { + ensure!( + !path.is_empty() && path.len() <= max && !path.contains(&0) && !path.starts_with(b"/"), + "invalid layer entry path" + ); + ensure!( + !path.split(|&b| b == b'/').any(|p| p == b".."), + "layer entry path escapes image root" + ); + ensure!( + root_directory || path.split(|&b| b == b'/').any(|p| !p.is_empty() && p != b"."), + "invalid root file entry" + ); + Ok(()) +} +fn parse_pax(mut bytes: &[u8]) -> Result> { + let mut result = BTreeMap::new(); + while !bytes.is_empty() { + let space = bytes.iter().position(|&b| b == b' ').context("invalid PAX record")?; + ensure!(space > 0 && space <= 10, "invalid PAX length"); + let length = std::str::from_utf8(&bytes[..space])?.parse::()?; + ensure!( + length > space + 2 && length <= bytes.len() && bytes[length - 1] == b'\n', + "invalid PAX record length" + ); + let record = std::str::from_utf8(&bytes[space + 1..length - 1])?; + let (key, value) = record.split_once('=').context("invalid PAX property")?; + ensure!( + key.len() <= 256 + && (matches!( + key, + "path" | "linkpath" | "size" | "mtime" | "atime" | "ctime" | "uid" | "gid" | "uname" | "gname" + ) || key.starts_with("SCHILY.xattr.")), + "unsupported sparse or unknown PAX property" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none() && result.len() <= 128, + "duplicate or excessive PAX properties" + ); + bytes = &bytes[length..]; + } + Ok(result) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/layers/tests.rs b/crates/oci/src/layers/tests.rs new file mode 100644 index 00000000000..044ec7bb43e --- /dev/null +++ b/crates/oci/src/layers/tests.rs @@ -0,0 +1,130 @@ +use super::*; +use std::io::Write; + +fn archive() -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_ustar(); + header.set_path("usr/bin/main").unwrap(); + header.set_size(4); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, &b"data"[..]).unwrap(); + builder.into_inner().unwrap() +} +fn descriptor(bytes: &[u8], media: &str) -> Descriptor { + Descriptor { + digest: crate::sha256(bytes), + size: bytes.len() as u64, + media_type: media.into(), + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} +#[test] +fn verifies_plain_gzip_and_zstd_against_actual_expanded_bytes() { + let tar = archive(); + let diff = crate::sha256(&tar); + for (bytes, media) in [ + (tar.clone(), "application/vnd.oci.image.layer.v1.tar"), + (gzip(&tar), "application/vnd.oci.image.layer.v1.tar+gzip"), + ( + zstd::stream::encode_all(&tar[..], 1).unwrap(), + "application/vnd.oci.image.layer.v1.tar+zstd", + ), + ] { + let result = verify_layer(&bytes[..], &descriptor(&bytes, media), diff, LayerLimits::default()).unwrap(); + assert_eq!(result.uncompressed_tar_bytes, tar.len() as u64); + assert_eq!(result.entries, 1); + assert_eq!(result.regular_file_bytes, 4); + assert!( + VerifiedImageSize::from_layers(bytes.len() as u64, &[result]) + .unwrap() + .cache_reservation_bytes + > result.uncompressed_tar_bytes + ); + } +} +#[test] +fn decompression_bomb_is_stopped_before_declared_diff_id_can_be_trusted() { + let expanded = vec![0u8; 1024 * 1024]; + let compressed = gzip(&expanded); + let result = verify_layer( + &compressed[..], + &descriptor(&compressed, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&expanded), + LayerLimits { + max_uncompressed_bytes: 4096, + ..LayerLimits::default() + }, + ); + assert!(result.unwrap_err().to_string().contains("byte bound")); +} +#[test] +fn catches_digest_mismatch_truncation_and_entry_limit() { + let bytes = archive(); + let descriptor = descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"); + assert!(verify_layer(&bytes[..], &descriptor, crate::sha256(b"wrong"), LayerLimits::default()).is_err()); + assert!(verify_layer( + &bytes[..100], + &descriptor, + crate::sha256(&bytes), + LayerLimits::default() + ) + .is_err()); + assert!(verify_layer( + &bytes[..], + &descriptor, + crate::sha256(&bytes), + LayerLimits { + max_entries: 0, + ..LayerLimits::default() + } + ) + .is_err()); +} +#[test] +fn extension_size_is_bounded_before_allocating_and_sparse_metadata_is_rejected() { + let mut header = tar::Header::new_gnu(); + header.set_path("pax").unwrap(); + header.set_entry_type(tar::EntryType::XHeader); + header.set_size(1 << 30); + header.set_cksum(); + let bytes = header.as_bytes().to_vec(); + assert!(verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"), + crate::sha256(&bytes), + LayerLimits::default() + ) + .unwrap_err() + .to_string() + .contains("metadata entry")); + assert!(parse_pax(b"25 GNU.sparse.size=123456\n").is_err()); + assert!(validate_path(b"safe/../../host", 4096, false).is_err()); + assert!(validate_path(b"/absolute", 4096, false).is_err()); +} +#[test] +fn counts_concatenated_gzip_members_and_checks_pax_sizes() { + let tar = archive(); + let mut bytes = gzip(&tar); + bytes.extend(gzip(&tar)); + let mut both = tar.clone(); + both.extend(&tar); + let result = verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&both), + LayerLimits::default(), + ) + .unwrap(); + assert_eq!(result.entries, 2); + assert_eq!(parse_pax(b"10 size=4\n").unwrap()["size"], "4"); + assert!(parse_pax(b"99 size=4\n").is_err()); +} diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs new file mode 100644 index 00000000000..aa1bc63f19b --- /dev/null +++ b/crates/oci/src/lib.rs @@ -0,0 +1,377 @@ +//! Validate immutable OCI objects before accepting a container deployment. +//! +//! Registry references and index annotations are discovery inputs. Only verified +//! object bytes and an exact selected platform establish the published image. + +pub mod layers; + +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, MAX_EXEC_STRING_BYTES}; +use std::collections::BTreeMap; + +pub const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +pub const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +pub const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +pub const DOCKER_MANIFEST: &str = "application/vnd.docker.distribution.manifest.v2+json"; +pub const DOCKER_INDEX: &str = "application/vnd.docker.distribution.manifest.list.v2+json"; +pub const DOCKER_CONFIG: &str = "application/vnd.docker.container.image.v1+json"; +pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_CONFIG_BYTES: usize = 1024 * 1024; +pub const MAX_LAYERS: usize = 256; +pub const MAX_INDEX_ENTRIES: usize = 256; +pub const MAX_IMAGE_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Descriptor { + pub media_type: String, + pub digest: OciDigest, + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub urls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_type: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Platform { + pub os: String, + pub architecture: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variant: Option, + #[serde(default, rename = "os.version", skip_serializing_if = "Option::is_none")] + pub os_version: Option, + #[serde(default, rename = "os.features", skip_serializing_if = "Vec::is_empty")] + pub os_features: Vec, +} + +impl Platform { + fn matches(&self, requested: &ImagePlatform) -> bool { + self.os == requested.os + && self.architecture == requested.architecture + && self.os_version.as_deref().is_none_or(str::is_empty) + && self.os_features.is_empty() + && matches!( + (self.architecture.as_str(), self.variant.as_deref()), + (_, None | Some("")) | ("arm64", Some("v8")) + ) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub schema_version: u32, + pub media_type: String, + pub config: Descriptor, + pub layers: Vec, + #[serde(default)] + artifact_type: Option, + #[serde(default)] + subject: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImageIndex { + pub schema_version: u32, + pub media_type: String, + pub manifests: Vec, +} + +pub fn sha256(bytes: &[u8]) -> OciDigest { + OciDigest::sha256(Sha256::digest(bytes).into()) +} + +pub fn verify_object(descriptor: &Descriptor, bytes: &[u8]) -> Result<()> { + ensure!( + descriptor.size == bytes.len() as u64, + "OCI object length differs from its descriptor" + ); + ensure!( + sha256(bytes) == descriptor.digest, + "OCI object SHA-256 differs from its descriptor" + ); + Ok(()) +} + +fn validate_descriptor(descriptor: &Descriptor, max_bytes: u64) -> Result<()> { + ensure!( + descriptor.size > 0 && descriptor.size <= max_bytes, + "OCI descriptor size exceeds admission bounds" + ); + ensure!( + descriptor.urls.is_empty(), + "external OCI descriptor URLs are not supported" + ); + ensure!(descriptor.data.is_none(), "inline OCI descriptor data is not supported"); + ensure!( + descriptor.artifact_type.is_none(), + "OCI artifacts are not executable images" + ); + Ok(()) +} + +pub fn parse_manifest(bytes: &[u8]) -> Result { + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI manifest is too large"); + let manifest: Manifest = serde_json::from_slice(bytes).context("invalid OCI image manifest")?; + ensure!(manifest.schema_version == 2, "unsupported OCI manifest schema version"); + ensure!( + matches!(manifest.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "unsupported image manifest media type" + ); + ensure!( + manifest.artifact_type.is_none() && manifest.subject.is_none(), + "OCI artifact manifests are not executable images" + ); + ensure!(manifest.layers.len() <= MAX_LAYERS, "too many OCI image layers"); + validate_descriptor(&manifest.config, MAX_CONFIG_BYTES as u64)?; + ensure!( + matches!(manifest.config.media_type.as_str(), OCI_CONFIG | DOCKER_CONFIG), + "unsupported image config media type" + ); + let mut total = manifest.config.size; + for layer in &manifest.layers { + validate_descriptor(layer, MAX_IMAGE_BYTES)?; + ensure!( + matches!( + layer.media_type.as_str(), + "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.v1.tar+zstd" + | "application/vnd.docker.image.rootfs.diff.tar" + | "application/vnd.docker.image.rootfs.diff.tar.gzip" + ), + "unsupported or foreign image layer media type" + ); + total = total.checked_add(layer.size).context("OCI image size overflow")?; + } + ensure!(total <= MAX_IMAGE_BYTES, "OCI image exceeds compressed object quota"); + Ok(manifest) +} + +/// Select exactly one executable image for the requested platform. BuildKit may +/// include attestation descriptors for unknown/unknown; they are never executed. +pub fn select_platform(bytes: &[u8], requested: &ImagePlatform) -> Result { + validate_platform(requested)?; + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI image index is too large"); + let index: ImageIndex = serde_json::from_slice(bytes).context("invalid OCI image index")?; + ensure!( + index.schema_version == 2 && matches!(index.media_type.as_str(), OCI_INDEX | DOCKER_INDEX), + "unsupported image index format" + ); + ensure!( + index.manifests.len() <= MAX_INDEX_ENTRIES, + "too many image index entries" + ); + let mut selected = None; + for descriptor in index.manifests { + if !descriptor.platform.as_ref().is_some_and(|p| p.matches(requested)) { + continue; + } + validate_descriptor(&descriptor, MAX_MANIFEST_BYTES as u64)?; + ensure!( + matches!(descriptor.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "selected platform is not an image manifest" + ); + ensure!( + selected.replace(descriptor).is_none(), + "OCI index has ambiguous images for the selected platform" + ); + } + selected.context("OCI image does not contain the selected Linux platform") +} + +fn validate_platform(platform: &ImagePlatform) -> Result<()> { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "unsupported container platform" + ); + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ImageConfig { + pub architecture: String, + pub os: String, + #[serde(default)] + pub variant: Option, + #[serde(default)] + pub config: ContainerConfig, + pub rootfs: RootFs, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerConfig { + #[serde(default)] + pub entrypoint: Option>, + #[serde(default)] + pub cmd: Option>, + #[serde(default)] + pub user: String, + #[serde(default, rename = "WorkingDir")] + pub working_directory: String, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub volumes: Option>, +} + +impl std::fmt::Debug for ContainerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerConfig") + .field("entrypoint", &self.entrypoint) + .field("cmd", &self.cmd) + .field("user", &self.user) + .field("working_directory", &self.working_directory) + .field( + "env_keys", + &self.env.as_ref().map(|env| { + env.iter() + .map(|v| v.split('=').next().unwrap_or("")) + .collect::>() + }), + ) + .field("volumes", &self.volumes.as_ref().map(|v| v.keys().collect::>())) + .finish() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RootFs { + #[serde(rename = "type")] + pub kind: String, + pub diff_ids: Vec, +} + +pub fn parse_config(bytes: &[u8], manifest: &Manifest, platform: &ImagePlatform) -> Result { + validate_platform(platform)?; + ensure!(bytes.len() <= MAX_CONFIG_BYTES, "OCI image config is too large"); + verify_object(&manifest.config, bytes)?; + let image: ImageConfig = serde_json::from_slice(bytes).context("invalid OCI image config")?; + ensure!( + Platform { + os: image.os.clone(), + architecture: image.architecture.clone(), + variant: image.variant.clone(), + os_version: None, + os_features: vec![] + } + .matches(platform), + "image config platform does not match selected platform" + ); + ensure!( + image.rootfs.kind == "layers" && image.rootfs.diff_ids.len() == manifest.layers.len(), + "image rootfs does not match layer descriptors" + ); + ensure!( + image.config.volumes.as_ref().is_none_or(BTreeMap::is_empty), + "image-declared volumes are unsupported in Stage 1" + ); + image.config.environment()?; + for argv in [&image.config.entrypoint, &image.config.cmd].into_iter().flatten() { + ensure!(argv.len() <= MAX_ARGV_ENTRIES, "too many image command arguments"); + for arg in argv { + validate_string(arg)?; + } + } + validate_string(&image.config.user)?; + validate_string(&image.config.working_directory)?; + ensure!( + image.config.working_directory.is_empty() || image.config.working_directory.starts_with('/'), + "image working directory must be absolute" + ); + Ok(image) +} + +fn validate_string(value: &str) -> Result<()> { + ensure!( + value.len() < MAX_EXEC_STRING_BYTES && !value.contains('\0'), + "invalid container startup string" + ); + Ok(()) +} + +impl ContainerConfig { + /// Keep image defaults separately from the normalized spec. Environment + /// values remain in the immutable image, never in hot deployment metadata. + pub fn environment(&self) -> Result> { + let env = self.env.as_deref().unwrap_or_default(); + ensure!(env.len() <= MAX_ENV_KEYS, "too many image environment variables"); + let mut result = BTreeMap::new(); + for value in env { + validate_string(value)?; + let (key, value) = value + .split_once('=') + .context("image environment entry must contain '='")?; + ensure!( + valid_env_key(key) && !key.starts_with("SPACETIMEDB_"), + "invalid or reserved image environment key" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate image environment key" + ); + } + Ok(result) + } + + pub fn argv(&self, override_command: Option<&[String]>) -> Result> { + let argv = match override_command { + Some(argv) => argv.to_vec(), + None => self + .entrypoint + .iter() + .flatten() + .chain(self.cmd.iter().flatten()) + .cloned() + .collect(), + }; + ensure!( + !argv.is_empty() && !argv[0].is_empty() && argv.len() <= MAX_ARGV_ENTRIES, + "image needs a nonempty main command" + ); + for arg in &argv { + validate_string(arg)?; + } + Ok(argv) + } +} + +pub fn valid_env_key(key: &str) -> bool { + let mut bytes = key.bytes(); + key.len() <= 256 + && bytes.next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +/// Descriptor closure used to pin all required objects before desired-state +/// commit. Reject a digest repeated with conflicting lengths or media types. +pub fn object_closure(manifest_descriptor: Descriptor, manifest: &Manifest) -> Result> { + validate_descriptor(&manifest_descriptor, MAX_MANIFEST_BYTES as u64)?; + let mut seen = BTreeMap::new(); + let mut objects = Vec::with_capacity(manifest.layers.len() + 2); + for descriptor in std::iter::once(manifest_descriptor) + .chain(std::iter::once(manifest.config.clone())) + .chain(manifest.layers.iter().cloned()) + { + match seen.insert(descriptor.digest, (descriptor.size, descriptor.media_type.clone())) { + Some(previous) if previous != (descriptor.size, descriptor.media_type.clone()) => { + bail!("OCI digest has conflicting descriptors") + } + Some(_) => {} + None => objects.push(descriptor), + } + } + Ok(objects) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/tests.rs b/crates/oci/src/tests.rs new file mode 100644 index 00000000000..6fe4192bfa2 --- /dev/null +++ b/crates/oci/src/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use serde_json::{json, Value}; + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + } +} +fn config() -> Value { + json!({"architecture":"arm64","os":"linux","config":{ + "Entrypoint":["node"],"Cmd":["server.js"],"User":"1000:1000","WorkingDir":"/app", + "Env":["PATH=/usr/bin","APP_KEY=image-secret"] + },"rootfs":{"type":"layers","diff_ids":[sha256(b"uncompressed layer")]}}) +} +fn fixture(config: &Value) -> (Vec, Vec, Descriptor) { + let config_bytes = serde_json::to_vec(config).unwrap(); + let manifest = serde_json::to_vec(&json!({ + "schemaVersion":2,"mediaType":OCI_MANIFEST, + "config":{"mediaType":OCI_CONFIG,"digest":sha256(&config_bytes),"size":config_bytes.len()}, + "layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":sha256(b"layer"),"size":5}] + })) + .unwrap(); + let descriptor = Descriptor { + media_type: OCI_MANIFEST.into(), + digest: sha256(&manifest), + size: manifest.len() as u64, + platform: Some(Platform { + os: "linux".into(), + architecture: "arm64".into(), + variant: Some("v8".into()), + os_version: None, + os_features: vec![], + }), + urls: vec![], + data: None, + artifact_type: None, + }; + (config_bytes, manifest, descriptor) +} + +#[test] +fn immutable_image_preserves_defaults_and_explicit_command_replaces_all_argv() { + let (bytes, raw, descriptor) = fixture(&config()); + verify_object(&descriptor, &raw).unwrap(); + let manifest = parse_manifest(&raw).unwrap(); + let image = parse_config(&bytes, &manifest, &platform()).unwrap(); + assert_eq!(image.config.argv(None).unwrap(), ["node", "server.js"]); + assert_eq!(image.config.argv(Some(&["/bin/sh".into()])).unwrap(), ["/bin/sh"]); + assert_eq!(image.config.user, "1000:1000"); + assert_eq!(image.config.working_directory, "/app"); + assert_eq!(image.config.environment().unwrap()["APP_KEY"], "image-secret"); + assert!(!format!("{image:?}").contains("image-secret")); + assert_eq!(object_closure(descriptor, &manifest).unwrap().len(), 3); +} + +#[test] +fn content_verification_checks_actual_sha256_and_length() { + assert_eq!( + sha256(b"abc").to_string(), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let (_, bytes, mut descriptor) = fixture(&config()); + assert!(verify_object(&descriptor, b"different").is_err()); + descriptor.size -= 1; + assert!(verify_object(&descriptor, &bytes).is_err()); +} + +#[test] +fn platform_selection_ignores_attestations_and_rejects_ambiguity_or_absence() { + let (_, _, descriptor) = fixture(&config()); + let mut attestation = descriptor.clone(); + attestation.platform = Some(Platform { + os: "unknown".into(), + architecture: "unknown".into(), + variant: None, + os_version: None, + os_features: vec![], + }); + let index = |descriptors: Vec| { + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":OCI_INDEX,"manifests":descriptors})).unwrap() + }; + assert_eq!( + select_platform(&index(vec![attestation.clone(), descriptor.clone()]), &platform()).unwrap(), + descriptor + ); + assert!(select_platform(&index(vec![descriptor.clone(), descriptor]), &platform()).is_err()); + assert!(select_platform(&index(vec![attestation]), &platform()).is_err()); +} + +#[test] +fn foreign_layers_urls_inline_data_and_artifacts_fail_closed() { + let (_, bytes, _) = fixture(&config()); + let baseline: Value = serde_json::from_slice(&bytes).unwrap(); + for (key, value) in [ + ("urls", json!(["http://169.254.169.254/latest/meta-data/"])), + ("data", json!("inline")), + ("artifactType", json!("application/test")), + ] { + let mut manifest = baseline.clone(); + manifest["layers"][0][key] = value; + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + } + let mut manifest = baseline.clone(); + manifest["layers"][0]["mediaType"] = json!("application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + let mut manifest = baseline; + manifest["artifactType"] = json!("application/test"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); +} + +#[test] +fn config_rejects_volumes_wrong_platform_rootfs_and_reserved_environment() { + for (field, value) in [ + ("Volumes", json!({"/escape":{}})), + ("Env", json!(["SPACETIMEDB_IDENTITY=forged"])), + ("Env", json!(["KEY=a", "KEY=b"])), + ("Env", json!(["NO_EQUALS"])), + ("Env", json!(["KEY=contains\u{0}nul"])), + ("WorkingDir", json!("relative/path")), + ] { + let mut config = config(); + config["config"][field] = value; + let (bytes, raw, _) = fixture(&config); + assert!( + parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err(), + "{field}" + ); + } + for (field, value) in [ + ("architecture", json!("amd64")), + ("rootfs", json!({"type":"layers","diff_ids":[]})), + ] { + let mut config = config(); + config[field] = value; + let (bytes, raw, _) = fixture(&config); + assert!(parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err()); + } +} + +#[test] +fn size_limits_and_conflicting_descriptors_are_enforced() { + let (_, bytes, descriptor) = fixture(&config()); + let mut manifest: Value = serde_json::from_slice(&bytes).unwrap(); + manifest["layers"][0]["size"] = json!(MAX_IMAGE_BYTES); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + assert!(parse_manifest(&vec![b' '; MAX_MANIFEST_BYTES + 1]).is_err()); + let mut manifest = parse_manifest(&bytes).unwrap(); + let mut conflicting = manifest.layers[0].clone(); + conflicting.size += 1; + manifest.layers.push(conflicting); + assert!(object_closure(descriptor, &manifest).is_err()); +} + +#[test] +fn empty_scratch_image_requires_explicit_main_command() { + let config = ContainerConfig::default(); + assert!(config.argv(None).is_err()); + assert!(config.argv(Some(&[])).is_err()); + assert!(config.argv(Some(&["".into()])).is_err()); + assert_eq!(config.argv(Some(&["/main".into()])).unwrap(), ["/main"]); + assert!(valid_env_key("_A0")); + assert!(!valid_env_key("0A")); + assert!(valid_env_key(&"A".repeat(256))); + assert!(!valid_env_key(&"A".repeat(257))); +} diff --git a/crates/pg/src/pg_server.rs b/crates/pg/src/pg_server.rs index f1de39f4efd..dcdb0ecc360 100644 --- a/crates/pg/src/pg_server.rs +++ b/crates/pg/src/pg_server.rs @@ -282,13 +282,11 @@ impl claims, - Err(err) => { - // TODO: Do not log the supplied password/token; then classify credential errors separately from provider failures. - log::warn!( - "PG: Authentication failed for identity `{}` on database {database}: {err}", - pwd.password - ); - let err = ErrorInfo::new("FATAL".to_owned(), "28P01".to_owned(), err.to_string()); + Err(_) => { + // The supplied password is a bearer token. Validator + // errors can also contain untrusted claims or responses. + log::warn!("PG: Authentication failed on database {database}"); + let err = ErrorInfo::new("FATAL".to_owned(), "28P01".to_owned(), "Invalid token".to_owned()); return close_client(client, err).await; } }; diff --git a/crates/smoketests/modules/Cargo.lock b/crates/smoketests/modules/Cargo.lock index 3bfe68ba8c2..d84b1fda1cf 100644 --- a/crates/smoketests/modules/Cargo.lock +++ b/crates/smoketests/modules/Cargo.lock @@ -1221,6 +1221,7 @@ dependencies = [ "spacetimedb-bindings-macro", "spacetimedb-primitives", "spacetimedb-sats", + "thiserror", ] [[package]] diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index fdf85987338..532d69176d3 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -32,6 +32,10 @@ directives = [ # Maximum number of JS procedure isolates per database. Omit to use the number # of cores reported by the OS. # procedure-instance-pool-size = 8 +# Wall-clock bound for each JS startup, schema description and function call. +# Defaults to 120 seconds; must be positive and no greater than 120 seconds. +# Timed-out reducers fail and roll back, including lifecycle calls during publish. +# execution-timeout = "120s" [v8-heap-policy] # Check the V8 heap after this many requests. Set to 0 to disable. diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 55c5795f533..f04329d9505 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -35,6 +35,8 @@ futures.workspace = true [dev-dependencies] env_logger.workspace = true +spacetimedb-auth.workspace = true +spacetimedb-datastore.workspace = true serial_test.workspace = true [lints] diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 09d35280562..54cc339d60d 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -68,6 +68,12 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Access the real standalone control/host environment for integration + /// tests that publish, migrate, or recover the running module. + pub fn environment(&self) -> &StandaloneEnv { + &self.env + } + /// Publish a complete configuration through the standalone control API. pub async fn republish_environment( &self, diff --git a/crates/testing/tests/deployment_publish.rs b/crates/testing/tests/deployment_publish.rs new file mode 100644 index 00000000000..7d894aa91fd --- /dev/null +++ b/crates/testing/tests/deployment_publish.rs @@ -0,0 +1,205 @@ +//! Exercise publication against the real host and Wasm module, including the +//! unchanged-program path and retries after a later module has been installed. +use serial_test::serial; +use spacetimedb::db::deployment::{current_deployment, install_publication_fence, DeploymentCommit}; +use spacetimedb::host::{FunctionArgs, UpdateDatabaseResult}; +use spacetimedb::messages::control_db::HostType; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::ST_DEPLOYMENT_OPERATION_ID; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Identity, Uuid}; +use spacetimedb_schema::auto_migrate::{MigrationPolicy, MigrationToken}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn prepared(bytes: &[u8], epoch: u64, previous: Option<&DeploymentCommit>, command: &str) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(epoch)), + publication_epoch: epoch, + publisher: Identity::ZERO, + expected_revision: previous.map(|request| request.deployment.revision().unwrap()), + expected_last_operation: previous.map(|request| request.operation_id), + prepared_manifest_hash: hash_bytes(epoch.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(bytes), + }), + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([3; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec![command.into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fence(module: &spacetimedb::host::ModuleHost, request: &DeploymentCommit) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); +} + +async fn confirmed(result: UpdateDatabaseResult) { + let (offset, durable) = match result { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. + } => (tx_offset, durable_offset), + other => panic!("expected a durable publication result, got {other:?}"), + }; + tokio::time::timeout(Duration::from_secs(10), async { + let offset = offset.await.unwrap(); + if let Some(mut durable) = durable { + durable.wait_for(offset).await.unwrap(); + } + }) + .await + .unwrap(); +} + +#[test] +#[serial] +fn container_only_changes_and_old_retries_preserve_the_current_module() { + let compiled = CompiledModule::compile("hosted-auth-test", CompilationMode::Debug); + let bytes = compiled.program_bytes(); + compiled.with_module_async(DEFAULT_CONFIG, |handle| async move { + let env = handle.environment(); + let database = env.get_database_by_identity(&handle.db_identity).await.unwrap().unwrap(); + let host = env.leader(database.id).await.unwrap(); + let module = host.module().await.unwrap(); + let first = prepared(&bytes, 1, None, "/app/first"); + fence(&module, &first); + // Even the first in-progress container publication fences the legacy + // raw API, including a byte-identical module update. + assert!(host.update(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible).await.is_err()); + // The rejected updater must restore the old host in its controller. + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&bytes)); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + confirmed(result).await; + let first_revision = first.deployment.revision().unwrap(); + let second = prepared(&bytes, 2, Some(&first), "/app/second"); + fence(&host.module().await.unwrap(), &second); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, second.clone()).await.unwrap()).await; + assert_ne!(first_revision, second.deployment.revision().unwrap()); + + // A valid custom section changes program bytes/hash without changing + // its definition. This exercises the actual Wasm migration and swap. + let mut newer_bytes = bytes.to_vec(); + newer_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let third = prepared(&newer_bytes, 3, Some(&second), "/app/third"); + fence(&host.module().await.unwrap(), &third); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, third.clone()).await.unwrap()).await; + let newest = third.deployment.revision().unwrap(); + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&newer_bytes)); + let before_retry = host.module().await.unwrap(); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + assert!(matches!(&result, UpdateDatabaseResult::DeploymentAlreadyCommitted { result, .. } if result.revision == first_revision)); + confirmed(result).await; + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, before_retry.info.module_hash); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + current.call_reducer(Identity::ZERO, None, None, None, None, "private_only", FunctionArgs::Nullary).await.unwrap().outcome.into_result().unwrap(); + + // A stale CAS and a forged association between bytes and declaration + // cannot change either component, and rejection leaves service usable. + let stale = prepared(&bytes, 4, Some(&first), "/app/stale"); + fence(¤t, &stale); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, stale).await.is_err()); + let mismatched = prepared(&bytes, 5, Some(&third), "/app/mismatch"); + fence(¤t, &mismatched); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, mismatched).await.is_err()); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + + let empty = spacetimedb::host::empty_module::program(spacetimedb::host::empty_module::VERSION).unwrap(); + let mut remove_module = prepared(&empty.bytes, 6, Some(&third), "/app/image-only"); + let DeploymentSpec::V1(spec) = &mut remove_module.deployment; + spec.module = ModuleComponent::SystemEmpty(spacetimedb_lib::deployment::system_empty::empty().descriptor); + // An init request cannot execute this instance's schema while storing + // the bytes of a different valid, prepared program. + assert!(current.init_database_with_deployment(empty.clone(), Some(remove_module.clone())).await.is_err()); + + // Removing a nonempty table fails during migration execution, after + // the tentative program and deployment/receipt writes. All roll back. + let observation_name = current.info.module_def.tables() + .find(|table| table.name.to_ascii_lowercase().contains("observation")) + .unwrap().name.to_string(); + let observation_table = current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<_> { + let table = tx.table_id_from_name(&observation_name)?.unwrap(); + tx.insert_via_serialize_bsatn(table, &product![ConnectionId::from_u128(700), Identity::ZERO, false, Identity::ZERO, false])?; + Ok(table) + }).unwrap(); + let policy = MigrationPolicy::BreakClients(MigrationToken { + database_identity: handle.db_identity, + old_module_hash: current.info.module_hash, + new_module_hash: empty.hash, + }.hash()); + fence(¤t, &remove_module); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, empty.bytes.clone(), policy.clone(), remove_module.clone()).await.unwrap(); + assert!(matches!(result, UpdateDatabaseResult::ErrorExecutingMigration(ref error) if error.to_string().contains("table contains data")), "{result:?}"); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + assert_eq!(current.relational_db().program().unwrap().unwrap().hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + assert_eq!(tx.table_row_count(observation_table), Some(1)); + }); + current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + tx.clear_table(observation_table)?; + Ok(()) + }).unwrap(); + // The same operation remains eligible after an execution rollback. + confirmed(host.update_with_deployment(database, HostType::Wasm, empty.bytes.clone(), policy, remove_module.clone()).await.unwrap()).await; + let current = host.module().await.unwrap(); + assert!(current.info.module_def.tables().next().is_none()); + assert!(spacetimedb::host::empty_module::matches_program(&spacetimedb_lib::deployment::system_empty::empty().descriptor, ¤t.relational_db().program().unwrap().unwrap())); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, remove_module.deployment.revision().unwrap()); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(4)); + }); + }); +} diff --git a/crates/testing/tests/hosted_invocation.rs b/crates/testing/tests/hosted_invocation.rs new file mode 100644 index 00000000000..269f3c582b6 --- /dev/null +++ b/crates/testing/tests/hosted_invocation.rs @@ -0,0 +1,285 @@ +//! Exercise Rust Wasm bindings and host admission using actual signed proofs. +use serial_test::serial; +use spacetimedb::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use spacetimedb::auth::invocation::InvocationCaller; +use spacetimedb::auth::JwtKeys; +use spacetimedb::db::deployment::install_container_fence; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerFenceRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, +}; +use spacetimedb_lib::sats::{product, AlgebraicValue, ProductValue}; +use spacetimedb_lib::{bsatn, ConnectionId, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime}; + +fn authenticate(source: Identity, target: Identity, lifetime: Duration) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 1, + grant_revision: 1, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + lifetime, + "wasm-integration", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn install_fence(module: &ModuleHost, source: Identity, generation: u64, allowed: bool) { + let db = module.relational_db(); + db.with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: generation, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + }, + )?; + Ok(()) + }) + .unwrap(); +} + +fn assert_connection_count(module: &ModuleHost, expected: u64) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(expected)); + } + Ok(()) + }) + .unwrap(); +} + +fn arguments(values: ProductValue) -> FunctionArgs { + FunctionArgs::Bsatn(bsatn::to_vec(&values).unwrap().into()) +} + +async fn call( + module: &ModuleHost, + caller: impl Into + Send, + connection: Option, + reducer: &str, + args: ProductValue, +) -> anyhow::Result<()> { + module + .call_reducer(caller, connection, None, None, None, reducer, arguments(args)) + .await? + .outcome + .into_result() +} + +#[test] +#[serial] +fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { + CompiledModule::compile("hosted-auth-test", CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |handle| async move { + let module = handle.client.module(); + let target = handle.db_identity; + let foreign = Identity::ONE; + let owner = Identity::ZERO; + let self_connection = ConnectionId::from_u128(101); + let foreign_connection = ConnectionId::from_u128(102); + let failed_disconnect = ConnectionId::from_u128(999); + let expired_connection = ConnectionId::from_u128(103); + install_fence(&module, target, 1, true); + install_fence(&module, foreign, 1, true); + let self_auth = authenticate(target, target, Duration::from_secs(30)); + let foreign_auth = authenticate(foreign, target, Duration::from_secs(30)); + // This in-process standalone fixture has no cloud reconciliation + // service. Grant rows alone must not open hosted admission. + assert!(!module.relational_db().hosted_admission().is_open()); + assert!(module + .call_identity_connected(self_auth.clone(), self_connection) + .await + .is_err()); + assert_connection_count(&module, 0); + module + .relational_db() + .hosted_admission() + .begin() + .unwrap() + .complete() + .unwrap(); + let expiring_auth = authenticate(target, target, Duration::from_secs(3)); + for (auth, connection) in [ + (self_auth.clone(), self_connection), + (foreign_auth.clone(), foreign_connection), + (self_auth.clone(), failed_disconnect), + (expiring_auth.clone(), expired_connection), + ] { + module.call_identity_connected(auth, connection).await.unwrap(); + } + assert_connection_count(&module, 4); + for (auth, sender, connection, internal) in [ + (&self_auth, target, self_connection, true), + (&foreign_auth, foreign, foreign_connection, false), + ] { + call( + &module, + auth, + Some(connection), + "inspect_context", + product![sender, Some(connection), internal, true], + ) + .await + .unwrap(); + let result = module + .call_procedure( + auth, + Some(connection), + None, + "inspect_procedure", + arguments(product![sender, Some(connection), internal]), + ) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); + for lifecycle in ["connected", "disconnected"] { + assert!(call(&module, auth, Some(connection), lifecycle, product![]) + .await + .is_err()); + } + } + // Ordinary owner calls and equal database identities remain external. + for ordinary in [target, owner] { + call( + &module, + ordinary, + None, + "inspect_context", + product![ordinary, Option::::None, false, false], + ) + .await + .unwrap(); + assert!(call(&module, ordinary, None, "internal_only", product![]) + .await + .is_err()); + } + call(&module, owner, None, "private_only", product![]).await.unwrap(); + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .unwrap(); + call(&module, &self_auth, Some(self_connection), "private_only", product![]) + .await + .unwrap(); + for reducer in ["internal_only", "private_only"] { + assert!( + call(&module, &foreign_auth, Some(foreign_connection), reducer, product![]) + .await + .is_err() + ); + } + call(&module, owner, None, "schedule_check", product![]).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = module + .call_procedure(owner, None, None, "scheduled_finished", FunctionArgs::Nullary) + .await; + if result.result.unwrap().return_val == AlgebraicValue::Bool(true) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("scheduled call did not observe internal authority"); + // Tokens remain cryptographically valid, but their persisted grants are revoked. + install_fence(&module, target, 2, false); + install_fence(&module, foreign, 2, false); + assert!(!module.relational_db().hosted_admission().is_open()); + // Reopening the database-wide gate must not restore a denied grant. + module + .relational_db() + .hosted_admission() + .begin() + .unwrap() + .complete() + .unwrap(); + assert!( + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .is_err() + ); + let result = module + .call_procedure( + &self_auth, + Some(self_connection), + None, + "inspect_procedure", + arguments(product![target, Some(self_connection), true]), + ) + .await; + assert!(result.result.is_err()); + if let Ok(remaining) = expiring_auth + .hosted + .as_ref() + .unwrap() + .expires_at() + .duration_since(SystemTime::now()) + { + tokio::time::sleep(remaining + Duration::from_millis(20)).await; + } + assert!(call( + &module, + &expiring_auth, + Some(expired_connection), + "internal_only", + product![] + ) + .await + .is_err()); + // Host cleanup retains captured flags and JWT sender after revocation and expiry. + for (sender, connection) in [ + (target, self_connection), + (foreign, foreign_connection), + (target, failed_disconnect), + (target, expired_connection), + ] { + module.call_identity_disconnected(sender, connection).await.unwrap(); + } + assert_connection_count(&module, 0); + for (connection, sender, internal, disconnected) in [ + (self_connection, target, true, true), + (foreign_connection, foreign, false, true), + (failed_disconnect, target, true, false), + (expired_connection, target, true, true), + ] { + call( + &module, + owner, + None, + "inspect_observation", + product![connection, sender, internal, disconnected], + ) + .await + .unwrap(); + } + }, + ); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index b1fc7fe9d9f..bff2694a5b8 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -239,6 +239,8 @@ Previously stored values are **not** defaults for the next publish. Every publis The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. +Managed publications also retain the complete resolved environment in the operation's private local `submission.json` file. Keep the publication directory private and out of source control. `--resume-publication` sends the original request bytes, including the original values, even if project files or shell variables have changed. Missing or altered retained input causes an error; resuming never substitutes an empty environment. When preserving the current module, the CLI checks its environment declarations against authenticated metadata for the selected database and program before creating this input. + For publishing from an HTTP client or a module procedure, see the [HTTP publish format and example](../../00300-resources/00200-reference/00200-http-api/00300-database.md#publishing-with-environment-values). Supply the module and complete environment in the request body; project configuration and shell overrides are CLI conveniences. ## Inspect published values diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 1908eef5f6c..9dd63aa41ba 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -29,6 +29,15 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime logout`↴](#spacetime-logout) * [`spacetime init`↴](#spacetime-init) * [`spacetime build`↴](#spacetime-build) +* [`spacetime container`↴](#spacetime-container) +* [`spacetime container url`↴](#spacetime-container-url) +* [`spacetime container exec`↴](#spacetime-container-exec) +* [`spacetime container logs`↴](#spacetime-container-logs) +* [`spacetime container build`↴](#spacetime-container-build) +* [`spacetime container status`↴](#spacetime-container-status) +* [`spacetime container start`↴](#spacetime-container-start) +* [`spacetime container stop`↴](#spacetime-container-stop) +* [`spacetime container restart`↴](#spacetime-container-restart) * [`spacetime server`↴](#spacetime-server) * [`spacetime server list`↴](#spacetime-server-list) * [`spacetime server set-default`↴](#spacetime-server-set-default) @@ -66,6 +75,7 @@ This document contains the help content for the `spacetime` command-line program * `logout` — * `init` — Initializes a new spacetime project. * `build` — Builds a spacetime module. +* `container` — Build and manage a database's container * `server` — Manage the connection to the SpacetimeDB server. WARNING: This command is UNSTABLE and subject to breaking changes. * `subscribe` — Subscribe to SQL queries on the database. WARNING: This command is UNSTABLE and subject to breaking changes. * `start` — Start a local SpacetimeDB instance @@ -138,6 +148,31 @@ Every publish replaces the complete declared environment. Put an env map in spac * `--env ` — Environment name for config file layering (e.g., dev, staging) * `--native-aot` — Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only) * `--dotnet-version ` — Target .NET SDK major version for C# projects (e.g. 8 or 10). Auto-detected when omitted. +* `--managed` — Use managed deployment publication, including for a module-only database +* `--container-platform ` — Required platform when publishing a container declaration + + Possible values: `linux/amd64`, `linux/arm64` + +* `--artifact-endpoint ` — Explicitly authorize this exact artifact URL to receive the publisher credential +* `--remove-container` — Explicitly remove the container while preserving the module unless separately changed +* `--remove-module` — Replace the module with the versioned empty module after migration preflight +* `--publication-state-dir ` — Private local directory retaining complete managed publication inputs and progress. The protected submission file includes resolved environment values. Keep this directory private; do not commit it or share it. Resume reuses these exact values without rereading project configuration or shell variables. +* `--resume-publication ` — Resume this operation directory without rebuilding or reading spacetime.json +* `--publication-wait ` — Seconds to wait for managed activation; pending operations retain their resume directory + + Default value: `60` +* `--buildkit-host ` — Explicit local BuildKit Unix socket for container source builds +* `--registry-auth-file ` — Explicit registry auth JSON; otherwise image preparation is anonymous +* `--build-secret ` — Build secret file, separate from runtime env_keys +* `--buildctl ` — Path to the buildctl executable for Dockerfile or Railpack builds + + Default value: `buildctl` +* `--railpack ` — Path to the Railpack executable for explicitly selected Railpack builds + + Default value: `railpack` +* `--skopeo ` — Path to the skopeo executable for copying prebuilt OCI images + + Default value: `skopeo` @@ -552,6 +587,208 @@ Builds a spacetime module. +## `spacetime container` + +Build and manage a database's container + +**Usage:** `spacetime container ` + +###### **Subcommands:** + +* `url` — Print a container's published HTTPS URL +* `exec` — Run a literal command in the current running container +* `logs` — Read retained stdout and stderr from one container attempt +* `build` — Prepare verified OCI artifacts locally without publishing +* `status` — Inspect container control state without opening its database +* `start` — Request container execution +* `stop` — Request container stop +* `restart` — Request a new container instance and environment snapshot + + + +## `spacetime container url` + +Print a container's published HTTPS URL + +**Usage:** `spacetime container url [OPTIONS] ` + +Discovery does not start the container or wait for readiness. No login is required. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `--port ` — Declared port name; required when several ports are published +* `-s`, `--server ` — The nickname, host name or URL of the server + + + +## `spacetime container exec` + +Run a literal command in the current running container + +**Usage:** `spacetime container exec [OPTIONS] -- ...` + +Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux, macOS, and Windows terminals are supported. Windows requires an attached VT-capable console for --tty; inherited asynchronous seekable files are unsupported. A lost connection does not establish that the process exited. + +###### **Arguments:** + +* `` — Database name or Identity +* `` — Executable and literal arguments; no shell expansion + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `-i`, `--interactive` — Forward stdin and send EOF when it closes +* `-t`, `--tty` — Allocate a PTY using this foreground terminal's dimensions +* `--workdir ` — Absolute working directory inside the container +* `-e`, `--env ` — Override a process environment variable; platform keys are reserved + + + +## `spacetime container logs` + +Read retained stdout and stderr from one container attempt + +**Usage:** `spacetime container logs [OPTIONS] ` + +A restart does not change the selected attempt. Without --json, stdout and stderr retain their original bytes and streams. Retention gaps and interrupted capture are reported on stderr. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--generation ` — Read this attempt; omitted selects the current generation once +* `--cursor ` — Resume after an opaque cursor returned by --json +* `-f`, `--follow` — Wait for further output from the selected attempt until it ends +* `--json` — Print one JSON page per line, including timestamps, streams, and resume cursors + + + +## `spacetime container build` + +Prepare verified OCI artifacts locally without publishing + +**Usage:** `spacetime container build [OPTIONS] --out-dir --platform [database]` + +###### **Arguments:** + +* `` — Database target in local spacetime.json; no server lookup + +###### **Options:** + +* `--project-path ` — Directory in which to find spacetime.json + + Default value: `.` +* `--out-dir ` — New directory for verified OCI artifacts and prepared.json +* `--platform ` — Target Linux platform, independent of this computer's architecture + + Possible values: `linux/amd64`, `linux/arm64` + +* `--env ` — Local configuration overlay name +* `--buildkit-host ` — Explicit local BuildKit Unix socket, required for source builds +* `--buildctl ` — BuildKit client executable + + Default value: `buildctl` +* `--railpack ` — Pinned Railpack executable for explicitly selected Railpack builds + + Default value: `railpack` +* `--skopeo ` — Skopeo executable for prebuilt registry images + + Default value: `skopeo` +* `--registry-auth-file ` — Explicit registry auth JSON; omitted means anonymous, never saved Docker credentials +* `--build-secret ` — Explicit build secret file; separate from runtime env_keys + + + +## `spacetime container status` + +Inspect container control state without opening its database + +**Usage:** `spacetime container status [OPTIONS] ` + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--json` — Print the typed response as JSON + + + +## `spacetime container start` + +Request container execution + +**Usage:** `spacetime container start [OPTIONS] ` + +Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--json` — Print the typed response as JSON +* `--request-id ` — Retry an original UUIDv7 request; DATABASE must be its recorded Identity + + + +## `spacetime container stop` + +Request container stop + +**Usage:** `spacetime container stop [OPTIONS] ` + +Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--json` — Print the typed response as JSON +* `--request-id ` — Retry an original UUIDv7 request; DATABASE must be its recorded Identity + + + +## `spacetime container restart` + +Request a new container instance and environment snapshot + +**Usage:** `spacetime container restart [OPTIONS] ` + +Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--json` — Print the typed response as JSON +* `--request-id ` — Retry an original UUIDv7 request; DATABASE must be its recorded Identity + + + ## `spacetime server` Manage the connection to the SpacetimeDB server. WARNING: This command is UNSTABLE and subject to breaking changes. diff --git a/modules/hosted-auth-test/Cargo.toml b/modules/hosted-auth-test/Cargo.toml new file mode 100644 index 00000000000..b37b9b44332 --- /dev/null +++ b/modules/hosted-auth-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hosted-auth-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/hosted-auth-test/src/lib.rs b/modules/hosted-auth-test/src/lib.rs new file mode 100644 index 00000000000..f6380f686eb --- /dev/null +++ b/modules/hosted-auth-test/src/lib.rs @@ -0,0 +1,141 @@ +//! Actual host integration fixture for verified container authentication. +use spacetimedb::{ConnectionId, Identity, ProcedureContext, ReducerContext, Table}; + +#[spacetimedb::table(accessor = observations)] +pub struct Observation { + #[primary_key] + connection: ConnectionId, + sender: Identity, + internal: bool, + jwt_identity: Identity, + disconnected: bool, +} + +#[spacetimedb::reducer(client_connected)] +pub fn connected(ctx: &ReducerContext) { + ctx.db.observations().insert(Observation { + connection: ctx.connection_id().unwrap(), + sender: ctx.sender(), + internal: ctx.sender_auth().is_internal(), + jwt_identity: ctx.sender_auth().jwt().unwrap().identity(), + disconnected: false, + }); +} + +#[spacetimedb::reducer(client_disconnected)] +pub fn disconnected(ctx: &ReducerContext) -> Result<(), String> { + let connection = ctx.connection_id().unwrap(); + let mut observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(ctx.sender(), observation.sender); + assert_eq!(ctx.sender_auth().is_internal(), observation.internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), observation.jwt_identity); + // Exercise host fallback cleanup after a user callback rejects disconnect. + if connection == ConnectionId::from_u128(999) { + return Err("intentional disconnect failure".into()); + } + observation.disconnected = true; + ctx.db.observations().connection().update(observation); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn inspect_context( + ctx: &ReducerContext, + sender: Identity, + connection: Option, + internal: bool, + jwt: bool, +) { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().has_jwt(), jwt); + if jwt { + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + } +} + +#[spacetimedb::reducer(internal)] +pub fn internal_only(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); +} + +#[spacetimedb::reducer(private)] +pub fn private_only(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn inspect_observation( + ctx: &ReducerContext, + connection: ConnectionId, + sender: Identity, + internal: bool, + disconnected: bool, +) { + let observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(observation.sender, sender); + assert_eq!(observation.jwt_identity, sender); + assert_eq!(observation.internal, internal); + assert_eq!(observation.disconnected, disconnected); +} + +#[spacetimedb::procedure] +pub fn inspect_procedure( + ctx: &mut ProcedureContext, + sender: Identity, + connection: Option, + internal: bool, +) -> bool { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + ctx.with_tx(|tx| { + assert_eq!(tx.sender(), sender); + assert_eq!(tx.connection_id(), connection); + assert_eq!(tx.sender_auth().is_internal(), internal); + assert_eq!(tx.sender_auth().jwt().unwrap().identity(), sender); + }); + true +} + +#[spacetimedb::table(accessor = scheduled_checks, scheduled(scheduled_check))] +pub struct ScheduledCheck { + #[primary_key] + #[auto_inc] + id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::reducer] +pub fn schedule_check(ctx: &ReducerContext) { + ctx.db.scheduled_checks().insert(ScheduledCheck { + id: 0, + scheduled_at: ctx.timestamp.into(), + }); +} + +#[spacetimedb::reducer] +pub fn scheduled_check(ctx: &ReducerContext, _job: ScheduledCheck) { + assert_eq!(ctx.sender(), ctx.database_identity()); + assert_eq!(ctx.connection_id(), None); + assert!(ctx.sender_auth().is_internal()); + assert!(!ctx.sender_auth().has_jwt()); + ctx.db.observations().insert(Observation { + connection: ConnectionId::from_u128(777), + sender: ctx.sender(), + internal: true, + jwt_identity: ctx.sender(), + disconnected: false, + }); +} + +#[spacetimedb::procedure] +pub fn scheduled_finished(ctx: &mut ProcedureContext) -> bool { + ctx.with_tx(|tx| { + tx.db + .observations() + .connection() + .find(ConnectionId::from_u128(777)) + .is_some() + }) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5ed3e919ac8..c5493c5c405 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: crates/bindings-typescript: dependencies: + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 base64-js: specifier: ^1.5.1 version: 1.5.1 @@ -98,6 +101,9 @@ importers: vue: specifier: ^3.3.0 version: 3.5.26(typescript@5.9.3) + ws: + specifier: ^8.18.3 + version: 8.18.3 devDependencies: '@angular/compiler': specifier: ^21.2.12 diff --git a/sdks/rust/Cargo.toml b/sdks/rust/Cargo.toml index 7f1f24f6ff7..24c52975e41 100644 --- a/sdks/rust/Cargo.toml +++ b/sdks/rust/Cargo.toml @@ -57,6 +57,9 @@ web-sys = { version = "0.3.77", features = ["HtmlDocument"], optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] home.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true # native-tls 0.2.17 fails to compile with Rust 1.93.0 due non-exhaustive diff --git a/sdks/rust/src/client_cache.rs b/sdks/rust/src/client_cache.rs index ab0a90980bd..afe8237ffa6 100644 --- a/sdks/rust/src/client_cache.rs +++ b/sdks/rust/src/client_cache.rs @@ -2,6 +2,9 @@ //! //! This module is internal, and may incompatibly change without warning. +#[cfg(test)] +mod uuid_tests; + use crate::callbacks::CallbackId; use crate::db_connection::{debug_log, PendingMutation, SharedCell}; use crate::spacetime_module::{InModule, SpacetimeModule, TableUpdate, WithBsatn}; @@ -449,7 +452,9 @@ impl TableHandle { /// See [`DbContextImpl::queue_mutation`]. fn queue_mutation(&self, mutation: PendingMutation) { - self.pending_mutations.unbounded_send(mutation).unwrap(); + // A retained table handle may outlive terminal connection cleanup. + // The closed queue rejects and releases its callback captures. + let _ = self.pending_mutations.unbounded_send(mutation); } /// Called by the autogenerated implementation of the [`crate::Table`] method of the same name. diff --git a/sdks/rust/src/client_cache/uuid_tests.rs b/sdks/rust/src/client_cache/uuid_tests.rs new file mode 100644 index 00000000000..a9516acc81b --- /dev/null +++ b/sdks/rust/src/client_cache/uuid_tests.rs @@ -0,0 +1,75 @@ +use super::*; +use spacetimedb_lib::Uuid; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CountedRow { + id: Uuid, + value: u64, + clones: Arc, +} +impl Clone for CountedRow { + fn clone(&self) -> Self { + self.clones.fetch_add(1, Ordering::SeqCst); + Self { + id: self.id, + value: self.value, + clones: self.clones.clone(), + } + } +} + +fn row(id: Uuid, value: u64, clones: &Arc) -> WithBsatn { + WithBsatn { + bsatn: spacetimedb_lib::bsatn::to_vec(&(id, value)).unwrap().into(), + row: CountedRow { + id, + value, + clones: clones.clone(), + }, + } +} + +#[test] +fn uuid_unique_lookup_clones_only_the_match_and_tracks_updates_and_deletes() { + let clones = Arc::new(AtomicUsize::new(0)); + let mut cache = TableCache::::new(None); + // This is the exact registration emitted by Rust codegen. + cache.add_unique_constraint::("id", |row| &row.id); + let ids = (0u8..128) + .map(|n| Uuid::from_random_bytes_v4([n; 16])) + .collect::>(); + cache.apply_diff(&TableUpdate { + inserts: ids + .iter() + .enumerate() + .map(|(n, id)| row(*id, n as u64, &clones)) + .collect(), + deletes: vec![], + }); + clones.store(0, Ordering::SeqCst); + // UniqueConstraintHandle::find delegates to this lookup and clones only + // its selected row. A whole-cache snapshot would increment this 128 times. + let found = cache.find_by_unique_index("id", &ids[63]).cloned().unwrap(); + assert_eq!((found.id, found.value), (ids[63], 63)); + assert_eq!(clones.load(Ordering::SeqCst), 1); + let absent = Uuid::from_random_bytes_v4([255; 16]); + assert!(cache.find_by_unique_index("id", &absent).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 1); + + cache.apply_diff(&TableUpdate { + inserts: vec![row(ids[63], 999, &clones)], + deletes: vec![row(ids[63], 63, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert_eq!(cache.find_by_unique_index("id", &ids[63]).cloned().unwrap().value, 999); + assert_eq!(clones.load(Ordering::SeqCst), 1); + cache.apply_diff(&TableUpdate { + inserts: vec![], + deletes: vec![row(ids[63], 999, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert!(cache.find_by_unique_index("id", &ids[63]).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 0); + assert_eq!(cache.find_by_unique_index("id", &ids[64]).unwrap().value, 64); +} diff --git a/sdks/rust/src/credentials.rs b/sdks/rust/src/credentials.rs index 1b4c5116af1..33828c975de 100644 --- a/sdks/rust/src/credentials.rs +++ b/sdks/rust/src/credentials.rs @@ -1,4 +1,8 @@ -//! Utilities for saving and re-using credentials. +//! Credentials for ordinary clients and hosted container processes. +//! +//! Hosted containers should use `Container` and +//! `DbConnectionBuilder::with_container_credentials` instead of saving +//! their short-lived tokens to a file. Those APIs are available in native builds. //! //! Users are encouraged to import this module by name and refer to its contents by qualified path, like: //! ```ignore @@ -8,6 +12,13 @@ //! } //! ``` +#[cfg(not(feature = "browser"))] +mod container; +#[cfg(all(test, not(feature = "browser")))] +pub(crate) use container::tests as container_tests; +#[cfg(not(feature = "browser"))] +pub use container::{Container, ContainerCredentialError, ContainerToken}; + #[cfg(not(feature = "browser"))] mod native_mod { use home::home_dir; diff --git a/sdks/rust/src/credentials/container.rs b/sdks/rust/src/credentials/container.rs new file mode 100644 index 00000000000..84784fba92b --- /dev/null +++ b/sdks/rust/src/credentials/container.rs @@ -0,0 +1,474 @@ +//! Explicit platform discovery and bounded, target-specific credential requests. + +use crate::Identity; +use http::Uri; +use reqwest::{header, redirect::Policy, Client, Response, Url}; +use serde::Deserialize; +use std::{ + fmt, + net::IpAddr, + path::{Component, PathBuf}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::time::Instant; + +const DATABASE_IDENTITY: &str = "SPACETIMEDB_DATABASE_IDENTITY"; +const SERVER_URI: &str = "SPACETIMEDB_SERVER_URI"; +const CREDENTIAL_BROKER: &str = "SPACETIMEDB_CREDENTIAL_BROKER"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_LIFETIME: Duration = Duration::from_secs(30); +const MAX_TOKEN: usize = 8192; +const MAX_BODY: usize = MAX_TOKEN + 256; +const MAX_HEADERS: usize = 4096; +const MAX_URI: usize = 4096; +// The Unix transport uses the same HTTP authority and request path as the +// guest proxy. This URL is never resolved or connected over TCP in Unix mode. +const LOCAL_HTTP_URI: &str = "http://127.0.0.1:18081/v1/credentials"; + +/// A credential error whose diagnostics never include discovery values, +/// response bodies, request headers, or tokens. +#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerCredentialError { + #[error("Missing or non-Unicode container discovery variable {0}")] + MissingEnvironment(&'static str), + #[error("Invalid container discovery configuration")] + InvalidDiscovery, + #[error("Unsupported container credential transport")] + UnsupportedTransport, + #[error("Container credentials cannot be combined with a static token")] + ConflictingCredentials, + #[error("Container credentials cannot be used with debug files that record tokens")] + DebugLogging, + #[error("Invalid container credential target")] + InvalidTarget, + #[error("Unable to resolve the database Identity on the selected server")] + NameResolution, + #[error("Container credential request was denied")] + Denied, + #[error("Container credential broker is unavailable")] + Unavailable, + #[error("Container credential request failed")] + Transport, + #[error("Container credential request exceeded its deadline")] + Timeout, + #[error("Invalid container credential response")] + InvalidResponse, + #[error("Container credential expired before the connection completed")] + Expired, +} + +type Result = std::result::Result; + +#[derive(Clone)] +enum Endpoint { + Http(Url), + Unix(PathBuf), +} + +/// Discovery configuration for a process running in a SpacetimeDB container. +/// +/// This value contains no credentials. Cloning it never copies a token. The +/// broker derives the sender from its authenticated runtime association, not +/// from the Identity in this configuration or from request fields. +/// +/// ```ignore +/// use spacetimedb_sdk::credentials; +/// let container = credentials::Container::from_env()?; +/// let connection = DbConnection::builder() +/// .with_container_credentials(container) +/// .build_async().await?; +/// ``` +/// +/// The builder defaults to this container's database and server. Use its +/// `with_uri` and `with_database_name` methods to select another database. +/// Explicit `self` resolves locally to [`Self::database_identity`]. +/// +/// Each builder connection requests a fresh token. To renew credentials and +/// reconnect with fresh subscriptions, use [`crate::ContainerSession`]. An +/// ordinary builder connection ends when its credential expires or is revoked. +/// Do not save the token returned by `on_connect` for later use. +#[derive(Clone)] +pub struct Container { + identity: Identity, + server_uri: Uri, + endpoint: Endpoint, +} + +impl fmt::Debug for Container { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Container").finish_non_exhaustive() + } +} + +impl Container { + /// Read only the three platform-injected discovery variables. Missing or + /// invalid configuration is an error; this never reads stored CLI/SDK + /// credentials, allocates an anonymous Identity, or chooses a default server. + /// No network or socket operation occurs until a token is requested. + pub fn from_env() -> Result { + let read = |key| std::env::var(key).map_err(|_| ContainerCredentialError::MissingEnvironment(key)); + Self::new(&read(DATABASE_IDENTITY)?, &read(SERVER_URI)?, &read(CREDENTIAL_BROKER)?) + } + + /// Use explicit discovery values instead of reading the environment. + /// + /// The broker must be an absolute `unix:///path` socket URI or an HTTP URI + /// with a numeric loopback address and the path `/v1/credentials`. Unix + /// sockets are supported on Unix targets. Broker redirects, proxies, DNS + /// lookup, embedded credentials, and fallback transports are forbidden. + pub fn new(database_identity: &str, server_uri: &str, broker_uri: &str) -> Result { + let identity = parse_identity(database_identity).map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + let server_uri = server_url(server_uri)? + .as_str() + .parse() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + Ok(Self { + identity, + server_uri, + endpoint: endpoint(broker_uri)?, + }) + } + + pub fn database_identity(&self) -> Identity { + self.identity + } + + pub fn server_uri(&self) -> &Uri { + &self.server_uri + } + + /// Obtain a fresh credential for HTTP or another client of `target`. + /// + /// Send it only to that database's trusted server. It is a bearer credential + /// with a lifetime of at most 30 seconds, possibly less. Keep it in memory; + /// never put it in environment variables, logs, images, or durable files. + /// This method neither retries a denial nor falls back to an owner token. + pub async fn token_for(&self, target: Identity) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.request_token(target)) + .await + .map_err(|_| ContainerCredentialError::Timeout)? + } + + async fn request_token(&self, target: Identity) -> Result { + let builder = client_builder(); + let (builder, uri) = match &self.endpoint { + Endpoint::Http(uri) => (builder, uri.clone()), + Endpoint::Unix(path) => { + #[cfg(unix)] + { + (builder.unix_socket(path.clone()), Url::parse(LOCAL_HTTP_URI).unwrap()) + } + #[cfg(not(unix))] + { + let _ = path; + return Err(ContainerCredentialError::UnsupportedTransport); + } + } + }; + let client = builder.build().map_err(|_| ContainerCredentialError::Transport)?; + // Identity::to_hex is fixed lowercase ASCII, so the body cannot contain + // request framing, another sender, or arbitrary JSON fields. + let body = format!("{{\"target_database\":\"{}\"}}", target.to_hex()); + let response = client + .post(uri) + .header(header::CONTENT_TYPE, "application/json") + .header(header::CONNECTION, "close") + .body(body) + .send() + .await + .map_err(transport_error)?; + match response.status().as_u16() { + 200 => {} + 401 | 403 => return Err(ContainerCredentialError::Denied), + 503 => return Err(ContainerCredentialError::Unavailable), + _ => return Err(ContainerCredentialError::InvalidResponse), + } + let headers = response.headers(); + if header_size(headers) > MAX_HEADERS + || headers.get_all(header::CONTENT_TYPE).iter().count() != 1 + || headers.get(header::CONTENT_TYPE).map(|value| value.as_bytes()) != Some(b"application/json") + || headers.get_all(header::CACHE_CONTROL).iter().count() != 1 + || headers.get(header::CACHE_CONTROL).map(|value| value.as_bytes()) != Some(b"no-store") + || headers.contains_key(header::TRANSFER_ENCODING) + || headers.contains_key(header::CONTENT_ENCODING) + || headers.get_all(header::CONTENT_LENGTH).iter().count() != 1 + { + return Err(ContainerCredentialError::InvalidResponse); + } + let length = headers[header::CONTENT_LENGTH] + .to_str() + .ok() + .filter(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .and_then(|value| value.parse::().ok()) + .filter(|length| *length > 0 && *length <= MAX_BODY) + .ok_or(ContainerCredentialError::InvalidResponse)?; + let bytes = bounded_body(response, length).await?; + if bytes.len() != length { + return Err(ContainerCredentialError::InvalidResponse); + } + let response: TokenResponse = + serde_json::from_slice(&bytes).map_err(|_| ContainerCredentialError::InvalidResponse)?; + let expiry = UNIX_EPOCH + .checked_add(Duration::from_secs(response.expires_unix_seconds)) + .ok_or(ContainerCredentialError::InvalidResponse)?; + let remaining = expiry + .duration_since(SystemTime::now()) + .ok() + .filter(|remaining| !remaining.is_zero() && *remaining <= MAX_LIFETIME) + .ok_or(ContainerCredentialError::InvalidResponse)?; + if response.token.is_empty() + || response.token.len() > MAX_TOKEN + || !response.token.bytes().all(|byte| byte.is_ascii_graphic()) + { + return Err(ContainerCredentialError::InvalidResponse); + } + // Retain a monotonic cap even if the system clock later moves backward. + let deadline = Instant::now() + remaining; + Ok(ContainerToken { + token: response.token, + target, + expiry, + deadline, + }) + } + + pub(crate) async fn prepare_connection(&self, uri: Option<&Uri>, target: Option<&str>) -> Result { + let (uri, target) = self.resolve_connection_target(uri, target).await?; + let credential = self.token_for(target).await?; + Ok(Prepared { + uri, + target: target.to_hex().to_string(), + credential, + }) + } + + pub(crate) async fn resolve_connection_target( + &self, + uri: Option<&Uri>, + target: Option<&str>, + ) -> Result<(Uri, Identity)> { + let uri = uri.unwrap_or(&self.server_uri); + let mut server = server_url(&uri.to_string())?; + let target = match target { + None | Some("self") => self.identity, + Some(target) => match parse_identity(target) { + Ok(identity) => identity, + Err(_) => { + // Preserve the SDK's server path prefix. Append the name as + // one encoded segment, never as request path syntax. + if target.is_empty() || target.len() > 256 || target.chars().any(char::is_control) { + return Err(ContainerCredentialError::InvalidTarget); + } + server + .path_segments_mut() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)? + .pop_if_empty() + .extend(["v1", "database", target, "identity"]); + resolve_name(server).await? + } + }, + }; + Ok((uri.clone(), target)) + } +} + +pub(crate) struct Prepared { + pub uri: Uri, + pub target: String, + pub credential: ContainerToken, +} + +/// An in-memory, target-specific bearer credential. Debug output is redacted. +/// Expiry is enforced by the receiving host even if its bytes are copied. +pub struct ContainerToken { + token: String, + target: Identity, + expiry: SystemTime, + deadline: Instant, +} + +impl ContainerToken { + /// The token for an `Authorization: Bearer ...` header. Do not log or save it. + pub fn as_str(&self) -> &str { + &self.token + } + + pub fn target(&self) -> Identity { + self.target + } + + pub fn expires_at(&self) -> SystemTime { + self.expiry + } + + pub fn remaining_lifetime(&self) -> Duration { + self.deadline.saturating_duration_since(Instant::now()) + } + + pub(crate) fn deadline(&self) -> Instant { + self.deadline + } +} + +impl fmt::Debug for ContainerToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ContainerToken") + .field("target", &self.target) + .field("expires_at", &self.expiry) + .field("token", &"") + .finish() + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TokenResponse { + token: String, + expires_unix_seconds: u64, +} + +fn client_builder() -> reqwest::ClientBuilder { + Client::builder() + .no_proxy() + .redirect(Policy::none()) + .referer(false) + .http1_only() + .pool_max_idle_per_host(0) + .connect_timeout(REQUEST_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() +} + +fn transport_error(error: reqwest::Error) -> ContainerCredentialError { + if error.is_timeout() { + ContainerCredentialError::Timeout + } else { + ContainerCredentialError::Transport + } +} + +async fn bounded_body(mut response: Response, limit: usize) -> Result> { + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(transport_error)? { + if chunk.len() > limit.saturating_sub(bytes.len()) { + return Err(ContainerCredentialError::InvalidResponse); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +fn header_size(headers: &header::HeaderMap) -> usize { + headers + .iter() + .map(|(key, value)| key.as_str().len() + value.as_bytes().len() + 4) + .sum() +} + +async fn resolve_name(uri: Url) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, async { + let client = client_builder() + .build() + .map_err(|_| ContainerCredentialError::NameResolution)?; + let response = client + .get(uri) + .send() + .await + .map_err(|_| ContainerCredentialError::NameResolution)?; + if response.status() != reqwest::StatusCode::OK || header_size(response.headers()) > MAX_HEADERS { + return Err(ContainerCredentialError::NameResolution); + } + let body = bounded_body(response, 64) + .await + .map_err(|_| ContainerCredentialError::NameResolution)?; + parse_identity(std::str::from_utf8(&body).map_err(|_| ContainerCredentialError::NameResolution)?) + .map_err(|_| ContainerCredentialError::NameResolution) + }) + .await + .map_err(|_| ContainerCredentialError::Timeout)? +} + +fn parse_identity(value: &str) -> Result { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(ContainerCredentialError::InvalidTarget); + } + Identity::from_hex(value).map_err(|_| ContainerCredentialError::InvalidTarget) +} + +fn parse_url(value: &str) -> Result { + if value.is_empty() || value.len() > MAX_URI || value.bytes().any(|byte| byte.is_ascii_control() || byte == b' ') { + return Err(ContainerCredentialError::InvalidDiscovery); + } + let uri = Url::parse(value).map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + if !uri.username().is_empty() || uri.password().is_some() || uri.query().is_some() || uri.fragment().is_some() { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(uri) +} + +fn server_url(value: &str) -> Result { + let mut uri = parse_url(value)?; + match uri.scheme() { + "http" | "https" => {} + "ws" => uri + .set_scheme("http") + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?, + "wss" => uri + .set_scheme("https") + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?, + _ => return Err(ContainerCredentialError::InvalidDiscovery), + } + if uri.host_str().is_none() { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(uri) +} + +fn endpoint(value: &str) -> Result { + let uri = parse_url(value)?; + match uri.scheme() { + "http" => { + // Parse the original authority as an IP, rejecting DNS aliases and + // URL parser shortcuts such as hexadecimal/abbreviated IPv4. + let original: Uri = value.parse().map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + let host = original.host().ok_or(ContainerCredentialError::InvalidDiscovery)?; + let ip: IpAddr = host + .trim_matches(['[', ']']) + .parse() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + if !ip.is_loopback() || uri.path() != "/v1/credentials" { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(Endpoint::Http(uri)) + } + "unix" => { + let path = value + .strip_prefix("unix://") + .ok_or(ContainerCredentialError::InvalidDiscovery)?; + if uri.host_str().is_some() + || uri.port().is_some() + || !path.starts_with('/') + || path.len() > 103 + || path.contains('%') + || path.split('/').any(|part| part == "." || part == "..") + || PathBuf::from(path) + .components() + .any(|part| !matches!(part, Component::RootDir | Component::Normal(_))) + { + return Err(ContainerCredentialError::InvalidDiscovery); + } + #[cfg(unix)] + return Ok(Endpoint::Unix(PathBuf::from(path))); + #[cfg(not(unix))] + return Err(ContainerCredentialError::UnsupportedTransport); + } + _ => Err(ContainerCredentialError::UnsupportedTransport), + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/sdks/rust/src/credentials/container/tests.rs b/sdks/rust/src/credentials/container/tests.rs new file mode 100644 index 00000000000..73a41f47338 --- /dev/null +++ b/sdks/rust/src/credentials/container/tests.rs @@ -0,0 +1,297 @@ +use super::*; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::oneshot, + task::JoinHandle, +}; + +pub(crate) const SOURCE: &str = "0000000000000000000000000000000000000000000000000000000000000001"; +const TARGET: &str = "0000000000000000000000000000000000000000000000000000000000000002"; + +pub(crate) fn token_body(token: &str, offset: u64) -> String { + let expiry = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + offset; + serde_json::json!({"token": token, "expires_unix_seconds": expiry}).to_string() +} + +pub(crate) fn response(status: u16, body: &str) -> Vec { + format!( + "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ).into_bytes() +} + +pub(crate) async fn read_request(socket: &mut (impl AsyncRead + Unpin)) -> String { + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + assert!(bytes.len() < MAX_HEADERS); + bytes.push(socket.read_u8().await.unwrap()); + } + let header = std::str::from_utf8(&bytes).unwrap(); + let length = header + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + assert!(length <= 4096); + let start = bytes.len(); + bytes.resize(start + length, 0); + socket.read_exact(&mut bytes[start..]).await.unwrap(); + String::from_utf8(bytes).unwrap() +} + +// Every endpoint is created by this test, explicitly uses numeric loopback, +// and has a retained task that callers join. No saved server/client config. +pub(crate) async fn http_fixture(responses: Vec>) -> (String, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + tokio::time::timeout(Duration::from_secs(3), async { + let mut requests = Vec::new(); + for response in responses { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + requests.push(read_request(&mut socket).await); + socket.write_all(&response).await.unwrap(); + socket.shutdown().await.unwrap(); + } + requests + }) + .await + .expect("owned credential fixture exceeded deadline") + }); + (uri, task) +} + +#[test] +fn discovery_rejects_nonlocal_ambiguous_and_credential_bearing_endpoints() { + for endpoint in [ + "http://example.invalid/v1/credentials", + "http://192.0.2.1/v1/credentials", + "http://localhost/v1/credentials", + "http://127.1/v1/credentials", + "http://0x7f000001/v1/credentials", + "http://127.0.0.1/wrong", + "http://127.0.0.1/v1/credentials?token=never-log-this", + "http://owner:never-log-this@127.0.0.1/v1/credentials", + "https://127.0.0.1/v1/credentials", + "unix://authority/socket", + "unix:relative.sock", + "unix:///run/../other.sock", + "unix:///run/%2e%2e/other.sock", + "unix:///run/credentials.sock#never-log-this", + "spacetimedb-unavailable:///credentials", + ] { + let error = Container::new(SOURCE, "https://server.invalid", endpoint).unwrap_err(); + assert!(!format!("{error:?} {error}").contains("never-log-this")); + } + assert!(Container::new(SOURCE, "https://owner:secret@server.invalid", LOCAL_HTTP_URI).is_err()); + assert!(Container::new("self", "https://server.invalid", LOCAL_HTTP_URI).is_err()); + let config = Container::new(SOURCE, "wss://server.invalid/prefix", LOCAL_HTTP_URI).unwrap(); + assert_eq!(config.database_identity(), parse_identity(SOURCE).unwrap()); + assert_eq!(config.server_uri().to_string(), "https://server.invalid/prefix"); +} + +#[tokio::test] +async fn each_request_fetches_a_fresh_target_specific_token_without_sender_fields() { + let (endpoint, server) = http_fixture(vec![ + response(200, &token_body("first-secret", 20)), + response(200, &token_body("second-secret", 20)), + ]) + .await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let target = parse_identity(TARGET).unwrap(); + let first = config.token_for(target).await.unwrap(); + let second = config.token_for(target).await.unwrap(); + assert_eq!(first.as_str(), "first-secret"); + assert_eq!(second.as_str(), "second-secret"); + assert_eq!(first.target(), target); + assert!(first.remaining_lifetime() <= MAX_LIFETIME); + assert!(!format!("{first:?}").contains("first-secret")); + for request in server.await.unwrap() { + assert!(request.starts_with("POST /v1/credentials HTTP/1.1\r\n")); + assert_eq!( + request.split_once("\r\n\r\n").unwrap().1, + format!("{{\"target_database\":\"{TARGET}\"}}") + ); + assert!(!request.to_ascii_lowercase().contains("authorization:")); + } +} + +#[tokio::test] +async fn response_validation_rejects_expiry_fields_tokens_and_framing_without_secret_diagnostics() { + let mut bad = vec![ + response(200, &token_body("", 20)), + response(200, &token_body("never-log-this\n", 20)), + response(200, &token_body(&"x".repeat(MAX_TOKEN + 1), 20)), + response(200, &token_body("never-log-this", 0)), + response(200, &token_body("never-log-this", 60)), + response(200, "{\"token\":\"never-log-this\",\"expires_unix_seconds\":1,\"sender\":\"forged\"}"), + response(200, "{\"token\":\"never-log-this\",\"token\":\"duplicate\",\"expires_unix_seconds\":1}"), + response(200, "{\"token\":null,\"expires_unix_seconds\":null}"), + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", MAX_BODY + 1).into_bytes(), + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: application/json\r\nCache-Control: no-store\r\n\r\n0\r\n\r\n".to_vec(), + ]; + let valid = String::from_utf8(response(200, &token_body("never-log-this", 20))).unwrap(); + bad.push( + valid + .replacen("\r\n", &format!("\r\nX-Padding: {}\r\n", "x".repeat(MAX_HEADERS)), 1) + .into_bytes(), + ); + bad.push(valid.replace("Cache-Control: no-store\r\n", "").into_bytes()); + bad.push( + valid + .replace("Content-Type: application/json", "Content-Type: text/plain") + .into_bytes(), + ); + for bytes in bad { + let (endpoint, server) = http_fixture(vec![bytes]).await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let error = config.token_for(parse_identity(TARGET).unwrap()).await.unwrap_err(); + assert!(matches!( + error, + ContainerCredentialError::InvalidResponse | ContainerCredentialError::Transport + )); + assert!(!format!("{error:?} {error}").contains("never-log-this")); + server.await.unwrap(); + } +} + +#[tokio::test] +async fn broker_denial_unavailability_and_redirects_never_fall_back() { + for (status, expected) in [ + (401, ContainerCredentialError::Denied), + (403, ContainerCredentialError::Denied), + (503, ContainerCredentialError::Unavailable), + (302, ContainerCredentialError::InvalidResponse), + ] { + let bytes = String::from_utf8(response(status, "never-log-this")) + .unwrap() + .replacen("\r\n", "\r\nLocation: https://must-not-contact.invalid/\r\n", 1) + .into_bytes(); + let (endpoint, server) = http_fixture(vec![bytes]).await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + assert_eq!( + config.token_for(parse_identity(TARGET).unwrap()).await.unwrap_err(), + expected + ); + assert_eq!(server.await.unwrap().len(), 1); + } +} + +#[tokio::test] +async fn cancelling_a_pending_request_closes_its_socket() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + read_request(&mut socket).await; + ready_tx.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read_u8()) + .await + .unwrap() + .unwrap_err() + .kind(), + std::io::ErrorKind::UnexpectedEof + ); + }); + let mut request = Box::pin(config.token_for(parse_identity(TARGET).unwrap())); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + _ = &mut request => panic!("stalled broker unexpectedly answered"), + ready = ready_rx => ready.unwrap(), + } + }) + .await + .unwrap(); + drop(request); + server.await.unwrap(); +} + +#[tokio::test] +async fn self_alias_stays_local_and_name_resolution_pins_the_returned_identity() { + let (broker, broker_task) = http_fixture(vec![ + response(200, &token_body("self-token", 20)), + response(200, &token_body("other-token", 20)), + ]) + .await; + let config = Container::new(SOURCE, "https://must-not-contact.invalid", &broker).unwrap(); + let own = config.prepare_connection(None, Some("self")).await.unwrap(); + assert_eq!(own.target, SOURCE); + let (resolver, resolver_task) = http_fixture(vec![response(200, TARGET)]).await; + let server: Uri = resolver.trim_end_matches("v1/credentials").parse().unwrap(); + let other = config.prepare_connection(Some(&server), Some("tasks")).await.unwrap(); + assert_eq!(other.target, TARGET); + assert_eq!(other.credential.target(), parse_identity(TARGET).unwrap()); + let requests = broker_task.await.unwrap(); + assert!(requests[0].ends_with(&format!("{{\"target_database\":\"{SOURCE}\"}}"))); + assert!(requests[1].ends_with(&format!("{{\"target_database\":\"{TARGET}\"}}"))); + assert!(resolver_task.await.unwrap()[0].starts_with("GET /v1/database/tasks/identity HTTP/1.1\r\n")); +} + +#[cfg(unix)] +#[tokio::test] +async fn unix_socket_uses_the_same_bounded_http_protocol_without_tcp_fallback() { + use std::{ + os::unix::fs::PermissionsExt, + sync::atomic::{AtomicUsize, Ordering}, + }; + use tokio::net::UnixListener; + static NEXT: AtomicUsize = AtomicUsize::new(0); + // Use an explicitly short base because ambient TMPDIR can exceed sockaddr_un + // limits on macOS. Atomic creation and private permissions retain ownership. + let directory = PathBuf::from("/tmp").join(format!( + "sdk-cred-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&directory).unwrap(); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700)).unwrap(); + struct Directory(PathBuf); + impl Drop for Directory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let directory = Directory(directory); + let socket = directory.0.join("broker.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let config = Container::new( + SOURCE, + "https://server.invalid", + &format!("unix://{}", socket.display()), + ) + .unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = read_request(&mut socket).await; + socket + .write_all(&response(200, &token_body("unix-secret", 20))) + .await + .unwrap(); + socket.shutdown().await.unwrap(); + request + }); + assert_eq!( + config + .token_for(parse_identity(TARGET).unwrap()) + .await + .unwrap() + .as_str(), + "unix-secret" + ); + let request = tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + assert!(request.to_ascii_lowercase().contains("host: 127.0.0.1:18081\r\n")); + // The listener is gone but its pathname is retained: failure must not fall + // back to the HTTP authority used to format requests on this transport. + assert!(config.token_for(parse_identity(TARGET).unwrap()).await.is_err()); +} diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 332aac1b322..c93e8e7ec73 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -31,6 +31,8 @@ use crate::{ websocket::{WsConnection, WsParams}, }; use bytes::Bytes; +#[cfg(not(feature = "browser"))] +use futures::FutureExt; use futures::StreamExt; #[cfg(feature = "browser")] use futures::{pin_mut, FutureExt}; @@ -55,6 +57,16 @@ use tokio::{ pub(crate) type SharedCell = Arc>; +#[cfg(all(test, not(feature = "browser")))] +mod builder_tests; + +#[cfg(not(feature = "browser"))] +pub(crate) mod container_session; +#[cfg(not(feature = "browser"))] +mod native_tasks; +#[cfg(not(feature = "browser"))] +use native_tasks::NativeTasks; + #[cfg(not(feature = "browser"))] type SharedAsyncCell = Arc>; #[cfg(feature = "browser")] @@ -136,6 +148,12 @@ impl DbContextImpl { /// applying its mutations to the client cache and invoking callbacks. fn process_message(&self, msg: ParsedMessage) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`process_message`: {msg:?}")); + if !self.is_active() && !matches!(&msg, ParsedMessage::Error(_)) { + // Local disconnect invalidated callbacks. Drain already received + // results without interpreting them as outstanding healthy calls; + // the socket's terminal event must still reach end_connection. + return Ok(()); + } match msg { // Error: route as a connection error if we never finished connecting, // otherwise treat it as an erroneous disconnect. @@ -317,17 +335,27 @@ impl DbContextImpl { /// /// Returns the terminal error that should be returned from `advance_*` methods. fn end_connection(&self, callback_error: Option) -> crate::Error { - let mut inner = self.inner.lock().unwrap(); let return_error = callback_error.clone().unwrap_or(crate::Error::Disconnected); + let (lifecycle, db_callbacks, mut subscriptions, connect_callback, disconnect_callback, connect_error_callback) = { + let mut inner = self.inner.lock().unwrap(); + let lifecycle = inner.connection_lifecycle; + inner.connection_lifecycle = ConnectionLifecycle::Ended; + ( + lifecycle, + std::mem::take(&mut inner.db_callbacks), + std::mem::take(&mut inner.subscriptions), + inner.on_connect.take(), + inner.on_disconnect.take(), + inner.on_connect_error.take(), + ) + }; - let lifecycle = inner.connection_lifecycle; - if lifecycle == ConnectionLifecycle::Ended { - return return_error; - } - inner.connection_lifecycle = ConnectionLifecycle::Ended; - - // Set `send_chan` to `None`, since `Self::is_active` checks that. - *self.send_chan.lock().unwrap() = None; + // Serialize request enqueueing with terminal closure. A retained + // DbConnection must not retain in-flight callback captures indefinitely. + let outgoing = self.send_chan.lock().unwrap().take(); + self.discard_pending_requests(); + drop((outgoing, db_callbacks, connect_callback)); + subscriptions.on_disconnect(&self.make_event_ctx(callback_error.clone())); match lifecycle { ConnectionLifecycle::Connecting => { @@ -335,26 +363,59 @@ impl DbContextImpl { source: InternalError::new("Connection closed before receiving the initial connection message"), }); let ctx: M::ErrorContext = self.make_event_ctx(Some(callback_error.clone())); - if let Some(connect_error_callback) = inner.on_connect_error.take() { + if let Some(connect_error_callback) = connect_error_callback { connect_error_callback(&ctx, callback_error.clone()); } callback_error } ConnectionLifecycle::Connected => { let ctx: M::ErrorContext = self.make_event_ctx(callback_error.clone()); - if let Some(disconnect_callback) = inner.on_disconnect.take() { + if let Some(disconnect_callback) = disconnect_callback { disconnect_callback(&ctx, callback_error.clone()); } - - // Call the `on_disconnect` method for all subscriptions. - inner.subscriptions.on_disconnect(&ctx); - return_error } ConnectionLifecycle::Ended => return_error, } } + fn discard_pending_requests(&self) { + let (reducer_callbacks, procedure_callbacks) = { + let mut inner = self.inner.lock().unwrap(); + ( + std::mem::take(&mut inner.reducer_callbacks), + std::mem::take(&mut inner.procedure_callbacks), + ) + }; + let mut queued = Vec::new(); + { + // There is exactly one supported advance_* caller, and get_message + // releases this guard before applying a terminal message. Avoid a + // blocking_lock here because advance_one_message_async runs in Tokio. + #[cfg(not(feature = "browser"))] + let mut pending = self + .pending_mutations_recv + .try_lock() + .expect("concurrent SDK message advancement"); + #[cfg(feature = "browser")] + let mut pending = self.pending_mutations_recv.lock().unwrap(); + pending.close(); + while let Ok(Some(mutation)) = pending.try_next() { + queued.push(mutation); + } + } + // Destructors may reenter SDK methods. Drop user captures only after + // releasing the connection and pending-queue locks. These calls have an + // unknown outcome; do not manufacture a successful reducer completion. + for mutation in queued { + if let PendingMutation::Subscribe { handle, .. } = &mutation { + handle.cancel_pending_callbacks(); + } + drop(mutation); + } + drop((reducer_callbacks, procedure_callbacks)); + } + fn make_event_ctx>(&self, event: E) -> Ctx { let imp = self.clone(); Ctx::new(imp, event) @@ -362,7 +423,9 @@ impl DbContextImpl { /// Apply all queued [`PendingMutation`]s. fn apply_pending_mutations(&self) -> crate::Result<()> { - while let Ok(Some(pending_mutation)) = get_lock_sync(&self.pending_mutations_recv).try_next() { + loop { + let pending = get_lock_sync(&self.pending_mutations_recv).try_next(); + let Ok(Some(pending_mutation)) = pending else { break }; self.apply_mutation(pending_mutation)?; } @@ -372,6 +435,16 @@ impl DbContextImpl { /// Apply an individual [`PendingMutation`]. fn apply_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`apply_mutation`: {mutation:?}")); + if !self.is_active() + && matches!( + &mutation, + PendingMutation::InvokeReducerWithCallback { .. } | PendingMutation::InvokeProcedureWithCallback { .. } + ) + { + // A call may have been queued behind the disconnect mutation. Drop + // its captures and continue driving the actual terminal event. + return Ok(()); + } match mutation { // Subscribe: register the subscription in the [`SubscriptionManager`] // and send the `Subscribe` WS message. @@ -491,6 +564,7 @@ impl DbContextImpl { // eventually resulting in disconnect callbacks being called // if the initial connection had completed. *self.send_chan.lock().unwrap() = None; + self.discard_pending_requests(); } // Callback stuff: these all do what you expect. @@ -605,13 +679,13 @@ impl DbContextImpl { // This may be unnecessary, but `tokio::select` does not document any ordering guarantees, // and if both `pending_mutations.next()` and `recv.next()` have values ready, // we want to process the pending mutation first. - if let Ok(pending_mutation) = pending_mutations.try_next() { - return Message::Local(pending_mutation.unwrap()); + if let Ok(Some(pending_mutation)) = pending_mutations.try_next() { + return Message::Local(pending_mutation); } #[cfg(not(feature = "browser"))] tokio::select! { - pending_mutation = pending_mutations.next() => Message::Local(pending_mutation.unwrap()), + Some(pending_mutation) = pending_mutations.next() => Message::Local(pending_mutation), incoming_message = recv.next() => Message::Ws(incoming_message), } @@ -621,7 +695,10 @@ impl DbContextImpl { pin_mut!(pending_fut, recv_fut); futures::select! { - pending_mutation = pending_fut => Message::Local(pending_mutation.unwrap()), + pending_mutation = pending_fut => match pending_mutation { + Some(pending_mutation) => Message::Local(pending_mutation), + None => Message::Ws(recv_fut.await), + }, incoming_message = recv_fut => Message::Ws(incoming_message), } } @@ -664,11 +741,9 @@ impl DbContextImpl { #[cfg(not(feature = "browser"))] pub fn run_threaded(&self) -> std::thread::JoinHandle<()> { let this = self.clone(); - std::thread::spawn(move || loop { - match this.advance_one_message_blocking() { - Ok(()) => (), - Err(e) if error_is_normal_disconnect(&e) => return, - Err(e) => panic!("{e:?}"), + std::thread::spawn(move || { + if let Err(e) = this.runtime.block_on(this.run_async()) { + panic!("{e:?}"); } }) } @@ -692,14 +767,66 @@ impl DbContextImpl { /// An async task which does [`Self::advance_one_message_async`] in a loop. /// + /// On native targets, completion also waits for the WebSocket and parser + /// tasks to finish. Dropping this future does not disconnect the connection; + /// call `disconnect` and keep driving it to completion to finish shutdown. + /// A cancelled wait may be resumed by calling this method again. + /// /// Called by the autogenerated `DbConnection` method of the same name. pub async fn run_async(&self) -> crate::Result<()> { - let this = self.clone(); + #[cfg(feature = "browser")] + { + self.run_until_disconnected().await + } + #[cfg(not(feature = "browser"))] + { + // Retain task ownership once per driver, not on every event-context + // clone. Recover a poisoned guard only to obtain the cleanup owner. + let background_tasks = Arc::clone( + &self + .inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .background_tasks, + ); + // Callback panics still propagate, but only after the native tasks + // have relinquished the socket and parser state. Catching an unwind + // outside run_async must not disguise a detached connection. + let result = std::panic::AssertUnwindSafe(self.run_until_disconnected()) + .catch_unwind() + .await; + // An outgoing queue failure can panic while holding this mutex. + // Recover only to close it; poisoning must not bypass task cleanup. + let outgoing = self.send_chan.lock().unwrap_or_else(|error| error.into_inner()).take(); + drop(outgoing); + let mut background_tasks = background_tasks.lock().await; + match &result { + Ok(Err(error)) => background_tasks.record_failure(error.clone()), + Err(_) => { + background_tasks.record_failure(InternalError::new("Connection event processing panicked").into()) + } + Ok(Ok(())) => (), + } + let joined = background_tasks.stop_and_join().await; + match result { + Ok(Ok(())) => joined, + Ok(Err(error)) => Err(error), + Err(panic) => std::panic::resume_unwind(panic), + } + } + } + + async fn run_until_disconnected(&self) -> crate::Result<()> { loop { - match this.advance_one_message_async().await { + match self.advance_one_message_async().await { Ok(()) => (), Err(e) if error_is_normal_disconnect(&e) => return Ok(()), + #[cfg(feature = "browser")] Err(e) => return Err(e), + // Errors applying a message or local mutation must also close + // the connection, including those not routed by process_message. + #[cfg(not(feature = "browser"))] + Err(e) => return Err(self.end_connection(Some(e))), } } } @@ -716,8 +843,7 @@ impl DbContextImpl { } self.pending_mutations_send .unbounded_send(PendingMutation::Disconnect) - .unwrap(); - Ok(()) + .map_err(|_| crate::Error::Disconnected) } /// Add a [`PendingMutation`] to the `pending_mutations` queue, @@ -725,8 +851,14 @@ impl DbContextImpl { /// /// This is used to defer operations which would otherwise need to hold a lock on `self.inner`, /// as otherwise running those operations within a callback would deadlock. - fn queue_mutation(&self, mutation: PendingMutation) { + fn queue_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { + let outgoing = self.send_chan.lock().unwrap(); + if outgoing.is_none() { + drop(outgoing); + return Err(crate::Error::Disconnected); + } self.pending_mutations_send.unbounded_send(mutation).unwrap(); + Ok(()) } /// Called by autogenerated table access methods. @@ -757,8 +889,7 @@ impl DbContextImpl { self.queue_mutation(PendingMutation::InvokeReducerWithCallback { reducer: reducer.into(), callback: Box::new(callback), - }); - Ok(()) + }) } /// Called by the autogenerated `DbConnection` method of the same name. @@ -788,7 +919,7 @@ impl DbContextImpl { + Send + 'static, ) { - self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { + let _ = self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { procedure: procedure_name, args: bsatn::to_vec(&args).expect("Failed to BSATN serialize procedure args"), callback: Box::new(move |ctx, ret| { @@ -823,6 +954,8 @@ enum ConnectionLifecycle { /// All the stuff in a [`DbContextImpl`] which can safely be locked while invoking callbacks. pub(crate) struct DbContextImplInner { + #[cfg(not(feature = "browser"))] + background_tasks: SharedAsyncCell, /// `Some` if not within the context of an outer runtime. The `Runtime` must /// then live as long as `Self`. #[allow(unused)] @@ -854,6 +987,9 @@ pub struct DbConnectionBuilder { token: Option, + #[cfg(not(feature = "browser"))] + container_credentials: Option, + on_connect: Option>, on_connect_error: Option>, on_disconnect: Option>, @@ -913,6 +1049,8 @@ impl DbConnectionBuilder { uri: None, database_name: None, token: None, + #[cfg(not(feature = "browser"))] + container_credentials: None, on_connect: None, on_connect_error: None, on_disconnect: None, @@ -953,6 +1091,25 @@ but you must call one of them, or else the connection will never progress. Ok(::new(imp)) } + /// Open a connection asynchronously on the current Tokio runtime. + /// + /// Unlike [`Self::build`], the WebSocket handshake does not block this task. + /// Dropping this future while the handshake is pending closes its socket + /// and releases the builder's callbacks. Callers can impose their own + /// connection deadline with [`tokio::time::timeout`]. Background connection + /// tasks are started only after the handshake succeeds. + /// + /// Requires an active Tokio runtime. As with [`Self::build`], the returned + /// connection must be advanced explicitly to receive events. + #[cfg(not(feature = "browser"))] + pub async fn build_async(self) -> crate::Result { + let handle = runtime::Handle::try_current().map_err(|error| { + InternalError::new("DbConnectionBuilder::build_async requires a Tokio runtime").with_cause(error) + })?; + let imp = self.build_native_impl(handle).await?; + Ok(::new(imp)) + } + #[cfg(feature = "browser")] pub async fn build(self) -> crate::Result { let imp = self.build_impl().await?; @@ -963,6 +1120,53 @@ but you must call one of them, or else the connection will never progress. /// to construct a [`DbContextImpl`]. #[cfg(not(feature = "browser"))] fn build_impl(self) -> crate::Result> { + let (runtime, handle) = enter_or_create_runtime()?; + // Keep an SDK-owned runtime outside its own block_on future. If the + // handshake fails, dropping that runtime inside the future would panic. + let imp = tokio::task::block_in_place(|| handle.block_on(self.build_native_impl(handle.clone())))?; + imp.inner.lock().unwrap().runtime = runtime; + Ok(imp) + } + + /// Share native construction between the synchronous and asynchronous API. + /// Credential acquisition and the handshake precede background connection + /// tasks. Cancelling either operation releases the pending connection. + #[cfg(not(feature = "browser"))] + async fn build_native_impl(mut self, handle: runtime::Handle) -> crate::Result> { + let credential = if let Some(container) = &self.container_credentials { + use crate::credentials::ContainerCredentialError; + // Reject before any socket or debug file is opened. Debug records + // include InitialConnection tokens; ordinary clients retain their + // existing behavior, but hosted credentials cannot be persisted. + if self.token.is_some() { + return Err(container_connect_error( + ContainerCredentialError::ConflictingCredentials, + )); + } + if self.additional_logging_path.is_some() { + return Err(container_connect_error(ContainerCredentialError::DebugLogging)); + } + let prepared = container + .prepare_connection(self.uri.as_ref(), self.database_name.as_deref()) + .await + .map_err(container_connect_error)?; + self.uri = Some(prepared.uri); + // Connect using the exact resolved Identity, so a later name + // reassignment cannot send this credential to another database. + self.database_name = Some(prepared.target); + Some(prepared.credential) + } else { + None + }; + self.build_native_with_credential(handle, credential).await + } + + #[cfg(not(feature = "browser"))] + async fn build_native_with_credential( + self, + handle: runtime::Handle, + credential: Option, + ) -> crate::Result> { let extra_logging = self .additional_logging_path .map(|path| { @@ -973,32 +1177,50 @@ but you must call one of them, or else the connection will never progress. .transpose()? .map(|file| Arc::new(StdMutex::new(file))); - let (runtime, handle) = enter_or_create_runtime()?; - let connection_id_override = get_connection_id_override(); - let ws_connection = tokio::task::block_in_place(|| { - handle.block_on(WsConnection::connect( - self.uri.unwrap(), - self.database_name.as_ref().unwrap(), - self.token.as_deref(), - connection_id_override, - self.params, - )) - }) - .map_err(|source| crate::Error::FailedToConnect { - source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), - })?; + let connect = WsConnection::connect( + self.uri.unwrap(), + self.database_name.as_ref().unwrap(), + credential + .as_ref() + .map(|token| token.as_str()) + .or(self.token.as_deref()), + connection_id_override, + self.params, + ); + let ws_connection = if let Some(credential) = &credential { + use crate::credentials::ContainerCredentialError; + let connection = tokio::time::timeout_at(credential.deadline(), connect) + .await + .map_err(|_| container_connect_error(ContainerCredentialError::Expired))? + // HTTP upgrade errors may contain response bodies/headers that + // echo secrets. Never retain those diagnostics for this mode. + .map_err(|_| container_connect_error(ContainerCredentialError::Transport))?; + if credential.remaining_lifetime().is_zero() { + return Err(container_connect_error(ContainerCredentialError::Expired)); + } + connection + } else { + connect.await.map_err(|source| crate::Error::FailedToConnect { + source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), + })? + }; - let (_websocket_loop_handle, raw_msg_recv, raw_msg_send) = + let (websocket_loop_handle, raw_msg_recv, raw_msg_send) = ws_connection.spawn_message_loop(&handle, extra_logging.clone()); - let (_parse_loop_handle, parsed_recv_chan) = - spawn_parse_loop::(raw_msg_recv, &handle, extra_logging.clone()); + let (parse_loop_handle, parsed_recv_chan) = spawn_parse_loop::(raw_msg_recv, &handle, extra_logging.clone()); let parsed_recv_chan = Arc::new(TokioMutex::new(parsed_recv_chan)); let (pending_mutations_send, pending_mutations_recv) = mpsc::unbounded(); let pending_mutations_recv = Arc::new(TokioMutex::new(pending_mutations_recv)); - let inner_ctx = build_db_ctx_inner(runtime, self.on_connect, self.on_connect_error, self.on_disconnect); + let inner_ctx = build_db_ctx_inner( + None, + NativeTasks::new(websocket_loop_handle, parse_loop_handle), + self.on_connect, + self.on_connect_error, + self.on_disconnect, + ); Ok(build_db_ctx( handle, inner_ctx, @@ -1085,6 +1307,28 @@ but you must call one of them, or else the connection will never progress. self } + /// Authenticate as the database backing this hosted container. + /// + /// Without explicit `with_uri` or `with_database_name` values, connect to + /// the database and server supplied by `container`. Other database names + /// are resolved on the selected server before requesting a target-specific + /// credential. The literal `self` resolves to the backing database Identity. + /// + /// Both `build` and `build_async` fetch a fresh credential on every call. + /// Failure never falls back to an anonymous Identity or saved owner token. + /// Combining this mode with a nonempty `with_token` or `with_debug_to_file` + /// is an error, regardless of the order of builder calls. + /// + /// The SDK does not automatically renew or reconnect this connection. + /// Handle expiration through `on_disconnect`, build a fresh connection, + /// and restore subscriptions as appropriate for your application. Tokens + /// returned through `on_connect` must not be saved or reused for reconnects. + #[cfg(not(feature = "browser"))] + pub fn with_container_credentials(mut self, container: crate::credentials::Container) -> Self { + self.container_credentials = Some(container); + self + } + /// Sets the compression used when a certain threshold in the message size has been reached. /// /// The current threshold used by the host is 1KiB for the entire server message @@ -1193,9 +1437,17 @@ Instead of registering multiple `on_disconnect` callbacks, register a single cal } } +#[cfg(not(feature = "browser"))] +fn container_connect_error(error: crate::credentials::ContainerCredentialError) -> crate::Error { + crate::Error::FailedToConnect { + source: InternalError::new("Failed to obtain or use container credentials").with_cause(error), + } +} + /// Create a [`DbContextImplInner`] wrapped in `Arc>`. fn build_db_ctx_inner( #[cfg(not(feature = "browser"))] runtime: Option, + #[cfg(not(feature = "browser"))] background_tasks: NativeTasks, on_connect_cb: Option>, on_connect_error_cb: Option>, @@ -1204,6 +1456,8 @@ fn build_db_ctx_inner( Arc::new(StdMutex::new(DbContextImplInner { #[cfg(not(feature = "browser"))] runtime, + #[cfg(not(feature = "browser"))] + background_tasks: Arc::new(TokioMutex::new(background_tasks)), db_callbacks: DbCallbacks::default(), reducer_callbacks: ReducerCallbacks::default(), @@ -1622,3 +1876,6 @@ pub(crate) fn next_query_set_id() -> QuerySetId { id: NEXT_QUERY_SET_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), } } + +#[cfg(all(test, not(feature = "browser")))] +mod terminal_tests; diff --git a/sdks/rust/src/db_connection/builder_tests.rs b/sdks/rust/src/db_connection/builder_tests.rs new file mode 100644 index 00000000000..0217d50c16d --- /dev/null +++ b/sdks/rust/src/db_connection/builder_tests.rs @@ -0,0 +1,239 @@ +use super::terminal_tests::bindings::RemoteModule; +use super::*; +use std::{net::SocketAddr, time::Duration}; +use tokio::{io::AsyncReadExt, net::TcpListener, sync::oneshot, task::JoinHandle}; + +async fn stalled_handshake_peer() -> (SocketAddr, oneshot::Receiver<()>, JoinHandle<()>) { + // This owned numeric-loopback listener never completes the HTTP upgrade. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (mut socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0, "client closed before sending the upgrade request"); + request.extend_from_slice(&buffer[..count]); + assert!(request.len() <= 16 * 1024); + } + ready_tx.send(()).unwrap(); + assert_eq!( + socket.read(&mut buffer).await.unwrap(), + 0, + "pending handshake socket was retained" + ); + }); + (address, ready_rx, peer) +} + +async fn check_pending_handshake_cleanup(use_timeout: bool) { + let (address, ready, peer) = stalled_handshake_peer().await; + let capture = Arc::new(()); + let weak_capture = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-builder-handshake-test") + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + result = &mut connect => panic!("stalled handshake unexpectedly completed: {:?}", result.err()), + result = ready => result.unwrap(), + } + }) + .await + .expect("async builder blocked the executor or never initiated its handshake"); + assert!(weak_capture.upgrade().is_some()); + + if use_timeout { + assert!(tokio::time::timeout(Duration::from_millis(30), connect).await.is_err()); + } else { + drop(connect); + } + assert!( + weak_capture.upgrade().is_none(), + "cancelled builder retained callback captures" + ); + tokio::time::timeout(Duration::from_secs(2), peer) + .await + .expect("cancelled handshake did not close its socket") + .unwrap(); +} + +// A current-thread runtime also proves the asynchronous builder does not use +// block_in_place or block_on while waiting for the peer's handshake response. +#[tokio::test] +async fn async_builder_timeout_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(true).await; +} + +#[tokio::test] +async fn async_builder_cancellation_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(false).await; +} + +#[test] +fn sync_builder_returns_connect_errors_without_dropping_its_runtime_inside_block_on() { + // The SDK rejects a query in the host URI before attempting any network I/O. + // Running outside Tokio exercises the synchronous API's owned runtime. + let result = DbConnectionBuilder::::new() + .with_uri("http://127.0.0.1:1/?unexpected=query") + .with_database_name("disposable-builder-error-test") + .build(); + assert!(matches!(result, Err(crate::Error::FailedToConnect { .. }))); +} + +#[tokio::test] +async fn container_builds_fetch_fresh_tokens_and_send_only_to_the_concrete_target() { + use crate::credentials::{container_tests as fixture, Container}; + use tokio::io::AsyncWriteExt; + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("first-private-token", 20)), + fixture::response(200, &fixture::token_body("second-private-token", 20)), + ]) + .await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let server_uri = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + for expected in ["first-private-token", "second-private-token"] { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = fixture::read_request(&mut socket).await; + assert!(request.starts_with(&format!("GET /v1/database/{}/subscribe?", fixture::SOURCE))); + assert!(request + .to_ascii_lowercase() + .contains(&format!("authorization: bearer {expected}\r\n"))); + // Deliberately echo secret material in a failed upgrade. Hosted + // connection errors must not retain the response or its headers. + socket.write_all(&fixture::response(403, expected)).await.unwrap(); + socket.shutdown().await.unwrap(); + } + }); + let container = Container::new(fixture::SOURCE, &server_uri, &broker).unwrap(); + for _ in 0..2 { + let error = DbConnectionBuilder::::new() + .with_container_credentials(container.clone()) + .with_database_name("self") + .build_async() + .await + .err() + .expect("fixture rejects the WebSocket upgrade"); + assert!(!format!("{error:?} {error}").contains("private-token")); + } + assert_eq!(broker_task.await.unwrap().len(), 2); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); +} + +#[tokio::test] +async fn container_auth_rejects_static_credentials_and_debug_files_before_io() { + use crate::credentials::{container_tests as fixture, Container}; + let container = Container::new( + fixture::SOURCE, + "https://must-not-contact.invalid", + "http://127.0.0.1:1/v1/credentials", + ) + .unwrap(); + let path = std::env::temp_dir().join(format!("must-not-write-sdk-credentials-{}", std::process::id())); + assert!(!path.exists()); + let result = DbConnectionBuilder::::new() + .with_token(Some("owner-secret")) + .with_container_credentials(container.clone()) + .build_async() + .await; + let error = result.err().unwrap(); + assert!(error.to_string().contains("static token")); + assert!(!format!("{error:?} {error}").contains("owner-secret")); + let result = DbConnectionBuilder::::new() + .with_container_credentials(container) + .with_debug_to_file(&path) + .build_async() + .await; + assert!(result.err().unwrap().to_string().contains("debug files")); + assert!(!path.exists()); +} + +#[tokio::test] +async fn cancelling_container_build_closes_broker_socket_and_releases_callbacks() { + use crate::credentials::{container_tests as fixture, Container}; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let container = Container::new(fixture::SOURCE, "https://must-not-contact.invalid", &broker).unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + ready_tx.send(()).unwrap(); + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let capture = Arc::new(()); + let weak = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_container_credentials(container) + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + _ = &mut connect => panic!("stalled broker unexpectedly answered"), + ready = ready_rx => ready.unwrap(), + } + }) + .await + .unwrap(); + drop(connect); + assert!(weak.upgrade().is_none()); + server.await.unwrap(); +} + +#[tokio::test] +async fn container_token_expiry_closes_a_stalled_websocket_handshake() { + use crate::credentials::{container_tests as fixture, Container}; + let (broker, broker_task) = + fixture::http_fixture(vec![fixture::response(200, &fixture::token_body("expiring-token", 1))]).await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let server_uri = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let container = Container::new(fixture::SOURCE, &server_uri, &broker).unwrap(); + let error = tokio::time::timeout( + Duration::from_secs(2), + DbConnectionBuilder::::new() + .with_container_credentials(container) + .build_async(), + ) + .await + .unwrap() + .err() + .unwrap(); + assert!(error.to_string().contains("expired")); + broker_task.await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); +} diff --git a/sdks/rust/src/db_connection/container_session.rs b/sdks/rust/src/db_connection/container_session.rs new file mode 100644 index 00000000000..e4500202e43 --- /dev/null +++ b/sdks/rust/src/db_connection/container_session.rs @@ -0,0 +1,621 @@ +//! Explicit ownership of successive container-authenticated connections. + +use super::{ConnectionLifecycle, DbConnectionBuilder, DbContextImpl, SpacetimeModule, WsParams}; +use crate::{ + credentials::{Container, ContainerCredentialError, ContainerToken}, + Identity, +}; +use futures::FutureExt; +use http::Uri; +use std::{ + panic::{resume_unwind, AssertUnwindSafe}, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::{Duration, SystemTime}, +}; +use tokio::{ + sync::Notify, + task::AbortHandle, + time::{sleep_until, Instant}, +}; + +#[cfg(test)] +mod tests; + +const RETRY_MIN: Duration = Duration::from_millis(250); +const RETRY_MAX: Duration = Duration::from_secs(5); +const MIN_EXTENSION: Duration = Duration::from_secs(1); +const REFRESH_MARGIN: Duration = Duration::from_secs(10); + +type Result = std::result::Result; +type Configure = Box) -> DbConnectionBuilder + Send>; + +/// One connection generation. The target is resolved once for the owner's +/// lifetime; reconnecting cannot follow a database name to another Identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ContainerSessionInfo { + pub generation: u64, + pub target: Identity, +} + +/// Why a managed connection is being closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionEndReason { + CredentialRenewal, + CredentialExpiry, + Disconnected, + Shutdown, + Failed, +} + +/// Unconfirmed reducer and procedure calls may already have committed. +/// The session does not track individual calls and never replays them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutstandingCallOutcomes { + Unknown, +} + +/// Redacted reason for a retry. No endpoint, token, or server response is retained. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionRetryReason { + Credential(ContainerCredentialError), + ExpiryNotExtended, + Connection, +} + +/// Session events contain no credentials. `Connected` means the server sent its +/// initial connection message. Subscriptions become ready separately, through +/// their normal `on_applied` callbacks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionEvent { + Connecting(ContainerSessionInfo), + Connected(ContainerSessionInfo), + /// The outgoing channel is sealed before this event is delivered. + Closing { + session: ContainerSessionInfo, + reason: ContainerSessionEndReason, + outstanding_calls: OutstandingCallOutcomes, + }, + /// Both native tasks have been joined before this event is delivered. + Closed { + session: ContainerSessionInfo, + reason: ContainerSessionEndReason, + outstanding_calls: OutstandingCallOutcomes, + }, + Retrying(ContainerSessionRetryReason), +} + +/// Redacted, terminal session errors. A denied credential is never retried or +/// replaced with an owner or anonymous credential. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum ContainerSessionError { + #[error("Invalid container session configuration")] + Configuration, + #[error("Container credential error: {0}")] + Credential(ContainerCredentialError), + #[error("Container session server returned a different sender Identity")] + IdentityMismatch, + #[error("Container session connection tasks failed during shutdown")] + Shutdown, + #[error("Container session application callback panicked")] + Panicked, + #[error("Container session generation counter exhausted")] + GenerationExhausted, +} + +/// Own successive connections authenticated through a container's broker. +/// +/// Pass a generated `DbConnection::builder()` with container discovery and an +/// optional server and database selection. The factory receives a fresh builder +/// for every attempt. Use it to create callbacks and recreate subscriptions in +/// `on_connect`. It must not set authentication, a target, or debug-file logging. +/// Put all callbacks in the factory, not in the initial builder. +/// +/// ```ignore +/// let mut session = spacetimedb_sdk::ContainerSession::new( +/// DbConnection::builder() +/// .with_container_credentials(spacetimedb_sdk::credentials::Container::from_env()?), +/// |generation, builder| builder.on_connect(move |conn, _, _| { +/// // Register callbacks and build new subscriptions for this generation. +/// conn.subscription_builder().subscribe("SELECT * FROM jobs"); +/// }), +/// )?.on_event(|event| { /* observe rotation and uncertain outstanding calls */ }); +/// tokio::select! { +/// result = session.run() => result?, +/// _ = shutdown_signal => {}, +/// } +/// session.shutdown_and_join().await?; +/// ``` +/// +/// `run` drives the connection and refreshes credentials before expiry. A token +/// that extends the lease causes a reconnect with an empty cache. There is an +/// interruption while the old connection shuts down and subscriptions rebuild. +/// The old WebSocket and parser are joined before the next WebSocket is opened. +/// Reducers and procedures are never replayed. Treat unconfirmed calls in a +/// closing generation as having unknown outcomes; use application request IDs +/// when an operation must be safely retried. +/// +/// Do not drive generated connection handles independently of this owner. +/// Handles supplied to callbacks belong to that generation and become inactive +/// when it closes. A new generation gets new callbacks, subscriptions and cache. +/// +/// Cancelling `run` pauses this owner, including renewal. Call `run` again to +/// resume, or call `shutdown_and_join` to finish shutdown. Cancelling the latter +/// is resumable. Dropping the owner aborts native tasks even if application code +/// retained a connection, but cannot wait for them; explicit shutdown is required +/// for completed cleanup. An application panic is propagated after task cleanup. +pub struct ContainerSession { + container: Container, + uri: Option, + database: Option, + target: Option<(Uri, Identity)>, + params: WsParams, + configure: Configure, + event: Option>, + current: Option>, + next_token: Option, + generation: u64, + retry_at: Option, + retry_delay: Duration, + stopped: bool, + failure: Option, +} + +struct Current { + context: DbContextImpl, + info: ContainerSessionInfo, + validity: Validity, + refresh_at: Instant, + signal: Arc, + connected_announced: bool, + closing: Option, + driver_finished: bool, + aborts: Vec, +} + +#[derive(Clone, Copy)] +struct Validity { + expiry: SystemTime, + deadline: Instant, +} + +impl Validity { + fn from_token(token: &ContainerToken) -> Self { + Self { + expiry: token.expires_at(), + deadline: token.deadline(), + } + } + + fn refresh_at(self) -> Instant { + let remaining = self.deadline.saturating_duration_since(Instant::now()); + self.deadline - REFRESH_MARGIN.min(remaining / 2) + } + + fn extended_by(self, new: Self) -> bool { + // The broker may issue another token with the same lease expiry. A + // fresh receipt time alone must not trigger endless reconnects. + new.expiry + .duration_since(self.expiry) + .is_ok_and(|delta| delta >= MIN_EXTENSION) + && new.deadline.saturating_duration_since(self.deadline) >= MIN_EXTENSION + } +} + +#[derive(Default)] +struct Signal { + // 0: waiting; 1: connected; 2: unexpected sender Identity. + state: AtomicU8, + notify: Notify, +} + +impl ContainerSession { + /// Create an owner without performing I/O. The initial builder supplies + /// discovery, target and WebSocket options. The factory supplies callbacks. + pub fn new( + mut initial: DbConnectionBuilder, + configure: impl FnMut(ContainerSessionInfo, DbConnectionBuilder) -> DbConnectionBuilder + Send + 'static, + ) -> Result { + if initial.token.is_some() + || initial.additional_logging_path.is_some() + || initial.on_connect.is_some() + || initial.on_disconnect.is_some() + || initial.on_connect_error.is_some() + { + return Err(ContainerSessionError::Configuration); + } + let container = initial + .container_credentials + .take() + .ok_or(ContainerSessionError::Configuration)?; + Ok(Self { + container, + uri: initial.uri, + database: initial.database_name, + target: None, + params: initial.params, + configure: Box::new(configure), + event: None, + current: None, + next_token: None, + generation: 0, + retry_at: None, + retry_delay: RETRY_MIN, + stopped: false, + failure: None, + }) + } + + /// Register a synchronous observer for redacted session events. + pub fn on_event(mut self, event: impl FnMut(ContainerSessionEvent) + Send + 'static) -> Self { + self.event = Some(Box::new(event)); + self + } + + /// Drive connections and credential renewal until shutdown or a terminal + /// error. This future owns no detached manager task and may be cancelled and + /// resumed. Calls made through each connection retain their normal semantics. + pub async fn run(&mut self) -> Result<()> { + let result = AssertUnwindSafe(async { + match self.run_loop().await { + Ok(()) => self.failure.map_or(Ok(()), Err), + Err(error) => { + self.failure.get_or_insert(error); + self.stopped = true; + self.next_token = None; + self.close_current(ContainerSessionEndReason::Failed); + self.finish_current().await?; + Err(self.failure.unwrap()) + } + } + }) + .catch_unwind() + .await; + match result { + Ok(result) => result, + Err(panic) => { + self.stopped = true; + self.next_token = None; + self.failure.get_or_insert(ContainerSessionError::Panicked); + // Retain the observer until cleanup finishes: even a callback + // capture's destructor must not bypass task ownership. + let observer = self.event.take(); + self.close_current(ContainerSessionEndReason::Failed); + let _ = AssertUnwindSafe(self.finish_current()).catch_unwind().await; + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| drop(observer))); + resume_unwind(panic) + } + } + } + + /// Stop renewal, seal outgoing calls and positively join the current native + /// connection. Safe to call again after cancellation or completed shutdown. + pub async fn shutdown_and_join(&mut self) -> Result<()> { + self.stopped = true; + self.next_token = None; + let result = AssertUnwindSafe(async { + self.close_current(ContainerSessionEndReason::Shutdown); + self.finish_current().await + }) + .catch_unwind() + .await; + match result { + Ok(result) => result.and_then(|()| self.failure.map_or(Ok(()), Err)), + Err(panic) => { + self.failure.get_or_insert(ContainerSessionError::Panicked); + let observer = self.event.take(); + let _ = AssertUnwindSafe(self.finish_current()).catch_unwind().await; + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| drop(observer))); + resume_unwind(panic) + } + } + } + + async fn run_loop(&mut self) -> Result<()> { + loop { + if self.current.as_ref().is_some_and(|current| current.closing.is_some()) { + self.finish_current().await?; + } + if self.stopped { + return self.failure.map_or(Ok(()), Err); + } + if self.current.is_none() { + if let Some(at) = self.retry_at { + sleep_until(at).await; + self.retry_at = None; + } + if self.target.is_none() { + // Store the concrete target before awaiting token issuance, + // including when a caller cancels during that request. + self.target = Some( + self.container + .resolve_connection_target(self.uri.as_ref(), self.database.as_deref()) + .await + .map_err(ContainerSessionError::Credential)?, + ); + } + let target = self.target.as_ref().unwrap().1; + let token = match self + .next_token + .take() + .filter(|token| !token.remaining_lifetime().is_zero()) + { + Some(token) => token, + None => match self.container.token_for(target).await { + Ok(token) => token, + Err(error) => { + self.credential_retry(error)?; + continue; + } + }, + }; + if !self.start(token).await? { + continue; + } + } + self.announce_connected()?; + let current = self.current.as_ref().unwrap(); + let context = current.context.clone(); + let signal = current.signal.clone(); + let deadline = current.validity.deadline; + let refresh_at = current.refresh_at; + tokio::select! { + biased; + result = AssertUnwindSafe(context.run_async()).catch_unwind() => self.driver_finished(result)?, + _ = signal.notify.notified() => {}, + _ = sleep_until(deadline) => self.close_current(ContainerSessionEndReason::CredentialExpiry), + _ = sleep_until(refresh_at) => self.refresh().await?, + } + } + } + + async fn start(&mut self, token: ContainerToken) -> Result { + self.generation = self + .generation + .checked_add(1) + .ok_or(ContainerSessionError::GenerationExhausted)?; + let (uri, target) = self.target.as_ref().unwrap().clone(); + let info = ContainerSessionInfo { + generation: self.generation, + target, + }; + let mut seed = DbConnectionBuilder::new(); + seed.params = self.params; + let mut builder = (self.configure)(info, seed); + if builder.uri.is_some() + || builder.database_name.is_some() + || builder.token.is_some() + || builder.container_credentials.is_some() + || builder.additional_logging_path.is_some() + { + return Err(ContainerSessionError::Configuration); + } + let signal = Arc::new(Signal::default()); + let connected_signal = signal.clone(); + let expected_sender = self.container.database_identity(); + let on_connect = builder.on_connect.take(); + builder.on_connect = Some(Box::new(move |connection, identity, token| { + if identity != expected_sender { + connected_signal.state.store(2, Ordering::Release); + } else { + if let Some(callback) = on_connect { + callback(connection, identity, token); + } + connected_signal.state.store(1, Ordering::Release); + } + connected_signal.notify.notify_one(); + })); + builder.uri = Some(uri); + builder.database_name = Some(target.to_hex().to_string()); + let validity = Validity::from_token(&token); + self.emit(ContainerSessionEvent::Connecting(info)); + let handle = tokio::runtime::Handle::current(); + let context = match builder.build_native_with_credential(handle, Some(token)).await { + Ok(context) => context, + Err(_) => { + self.retry(ContainerSessionRetryReason::Connection); + return Ok(false); + } + }; + // A new context has no driver yet. Its task owner cannot be contended. + // No suspension is allowed between construction and retaining Current. + let aborts = context + .inner + .lock() + .unwrap() + .background_tasks + .try_lock() + .unwrap() + .abort_handles(); + self.current = Some(Current { + context, + info, + validity, + refresh_at: validity.refresh_at(), + signal, + connected_announced: false, + closing: None, + driver_finished: false, + aborts, + }); + self.retry_at = None; + Ok(true) + } + + async fn refresh(&mut self) -> Result<()> { + let current = self.current.as_ref().unwrap(); + let context = current.context.clone(); + let signal = current.signal.clone(); + let deadline = current.validity.deadline; + let target = current.info.target; + tokio::select! { + biased; + result = AssertUnwindSafe(context.run_async()).catch_unwind() => self.driver_finished(result)?, + _ = signal.notify.notified() => {}, + _ = sleep_until(deadline) => self.close_current(ContainerSessionEndReason::CredentialExpiry), + result = self.container.token_for(target) => { + match result { + Ok(token) if self.current.as_ref().unwrap().validity.extended_by(Validity::from_token(&token)) => { + self.next_token = Some(token); + self.close_current(ContainerSessionEndReason::CredentialRenewal); + } + Ok(_) => self.retry(ContainerSessionRetryReason::ExpiryNotExtended), + Err(error) => self.credential_retry(error)?, + } + } + } + Ok(()) + } + + fn credential_retry(&mut self, error: ContainerCredentialError) -> Result<()> { + match error { + ContainerCredentialError::Unavailable + | ContainerCredentialError::Transport + | ContainerCredentialError::Timeout => { + self.retry(ContainerSessionRetryReason::Credential(error)); + Ok(()) + } + _ => Err(ContainerSessionError::Credential(error)), + } + } + + fn retry(&mut self, reason: ContainerSessionRetryReason) { + let at = Instant::now() + self.retry_delay; + self.retry_delay = (self.retry_delay * 2).min(RETRY_MAX); + if let Some(current) = self.current.as_mut() { + current.refresh_at = at; + } else { + self.retry_at = Some(at); + } + self.emit(ContainerSessionEvent::Retrying(reason)); + } + + fn announce_connected(&mut self) -> Result<()> { + let current = self.current.as_mut().unwrap(); + match current.signal.state.load(Ordering::Acquire) { + 2 => return Err(ContainerSessionError::IdentityMismatch), + 1 if !current.connected_announced => { + current.connected_announced = true; + self.retry_delay = RETRY_MIN; + let info = current.info; + self.emit(ContainerSessionEvent::Connected(info)); + } + _ => {} + } + Ok(()) + } + + fn driver_finished(&mut self, result: std::thread::Result>) -> Result<()> { + // run_async joins both native tasks before returning or unwinding. + self.current.as_mut().unwrap().driver_finished = true; + match result { + Err(panic) => resume_unwind(panic), + Ok(_) => { + // EOF can win the select immediately after InitialConnection. + // Check the observed sender before discarding this generation. + self.announce_connected()?; + self.close_current(ContainerSessionEndReason::Disconnected); + self.retry_at = Some(Instant::now() + self.retry_delay); + self.retry_delay = (self.retry_delay * 2).min(RETRY_MAX); + Ok(()) + } + } + } + + fn close_current(&mut self, reason: ContainerSessionEndReason) { + let Some(current) = self.current.as_mut() else { + return; + }; + if current.closing.is_some() { + return; + } + current.closing = Some(reason); + current.request_stop(); + let session = current.info; + self.emit(ContainerSessionEvent::Closing { + session, + reason, + outstanding_calls: OutstandingCallOutcomes::Unknown, + }); + } + + async fn finish_current(&mut self) -> Result<()> { + let Some(current) = self.current.as_ref() else { + return Ok(()); + }; + let reason = current.closing.expect("only a closing connection can be joined"); + let info = current.info; + let context = current.context.clone(); + if !current.driver_finished { + let tasks = context + .inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .background_tasks + .clone(); + let joined = tasks.lock().await.stop_and_join().await; + // A cancelled join is resumable through the retained NativeTasks. + self.current.as_mut().unwrap().driver_finished = true; + if joined.is_err() { + self.failure.get_or_insert(ContainerSessionError::Shutdown); + self.stopped = true; + } + } + // Complete normal SDK terminal cleanup only after native tasks joined. + // This discards queued and in-flight calls without replaying them or + // manufacturing success, including when a retained handle exists. + let terminal = std::panic::catch_unwind(AssertUnwindSafe(|| context.end_connection(None))); + self.current = None; + if let Err(panic) = terminal { + resume_unwind(panic); + } + self.emit(ContainerSessionEvent::Closed { + session: info, + reason, + outstanding_calls: OutstandingCallOutcomes::Unknown, + }); + self.failure.map_or(Ok(()), Err) + } + + fn emit(&mut self, event: ContainerSessionEvent) { + if let Some(observer) = self.event.as_mut() { + observer(event); + } + } +} + +impl Current { + fn request_stop(&self) { + // Seal calls synchronously, including handles retained by applications. + // Closing before InitialConnection has the ordinary disconnect semantics. + let mut inner = self.context.inner.lock().unwrap_or_else(|error| error.into_inner()); + if inner.connection_lifecycle == ConnectionLifecycle::Connecting { + inner.connection_lifecycle = ConnectionLifecycle::Ended; + } + drop(inner); + let outgoing = self + .context + .send_chan + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + drop(outgoing); + for task in &self.aborts { + task.abort(); + } + } +} + +impl Drop for ContainerSession { + fn drop(&mut self) { + if let Some(current) = &self.current { + current.request_stop(); + } + } +} diff --git a/sdks/rust/src/db_connection/container_session/tests.rs b/sdks/rust/src/db_connection/container_session/tests.rs new file mode 100644 index 00000000000..a39afc708f3 --- /dev/null +++ b/sdks/rust/src/db_connection/container_session/tests.rs @@ -0,0 +1,704 @@ +use super::super::{ + native_tasks::NativeTasks, + terminal_tests::bindings::{self, RemoteModule}, +}; +use super::*; +use crate::{credentials::container_tests as fixture, DbContext, Table}; +use bindings::{identity_connected, ConnectedTableAccess}; +use futures::{SinkExt, StreamExt}; +use spacetimedb_client_api_messages::websocket::{ + common::{BsatnRowList, RowSizeHint}, + v2 as ws, +}; +use spacetimedb_lib::{bsatn, ConnectionId}; +use std::{ + future::Future, + sync::{atomic::AtomicUsize, Mutex}, + time::UNIX_EPOCH, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{mpsc, oneshot}, + task::JoinHandle, + time::timeout, +}; +use tokio_tungstenite::{ + tungstenite::{handshake::server::Response, Message}, + WebSocketStream, +}; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct EmptyArgs {} +impl crate::spacetime_module::InModule for EmptyArgs { + type Module = RemoteModule; +} +use std::result::Result; + +struct Task(Option>); +impl Task { + fn spawn(body: impl Future + Send + 'static) -> Self + where + T: Send + 'static, + { + Self(Some(tokio::spawn(body))) + } + async fn join(mut self) -> T { + let result = timeout(Duration::from_secs(7), self.0.as_mut().unwrap()) + .await + .unwrap() + .unwrap(); + self.0 = None; + result + } +} +impl Drop for Task { + fn drop(&mut self) { + if let Some(task) = &self.0 { + task.abort(); + } + } +} + +#[derive(Default)] +struct PeerStats { + subscriptions: usize, + reducers: usize, + procedures: usize, +} + +async fn send(socket: &mut WebSocketStream, message: ws::ServerMessage) { + let mut bytes = vec![0]; + bytes.extend(bsatn::to_vec(&message).unwrap()); + socket.send(Message::Binary(bytes.into())).await.unwrap(); +} + +async fn peer(sessions: usize, sender: Identity) -> (String, mpsc::UnboundedReceiver, Task>) { + peer_with_close(sessions, sender, false).await +} + +async fn peer_with_close( + sessions: usize, + sender: Identity, + close_initially: bool, +) -> (String, mpsc::UnboundedReceiver, Task>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}", listener.local_addr().unwrap()); + let (applied, rx) = mpsc::unbounded_channel(); + let task = Task::spawn(async move { + timeout(Duration::from_secs(6), async { + let mut stats = Vec::new(); + for generation in 1..=sessions { + let (socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut socket = tokio_tungstenite::accept_hdr_async( + socket, + |request: &http::Request<()>, mut response: Response| { + assert!(request.uri().path().contains(fixture::SOURCE)); + assert!(request.headers()["authorization"] + .to_str() + .unwrap() + .starts_with("Bearer disposable-")); + response + .headers_mut() + .insert("sec-websocket-protocol", ws::BIN_PROTOCOL.parse().unwrap()); + Ok(response) + }, + ) + .await + .unwrap(); + send( + &mut socket, + ws::ServerMessage::InitialConnection(ws::InitialConnection { + identity: sender, + connection_id: ConnectionId::from_u128(generation as u128), + token: "disposable-server-token".into(), + }), + ) + .await; + let mut current = PeerStats::default(); + if close_initially { + socket.close(None).await.unwrap(); + stats.push(current); + continue; + } + while let Some(message) = socket.next().await { + let Ok(Message::Binary(bytes)) = message else { + break; + }; + match bsatn::from_slice::(&bytes).unwrap() { + ws::ClientMessage::Subscribe(subscribe) => { + current.subscriptions += 1; + let row = bsatn::to_vec(&bindings::Connected { identity: sender }).unwrap(); + send( + &mut socket, + ws::ServerMessage::SubscribeApplied(ws::SubscribeApplied { + request_id: subscribe.request_id, + query_set_id: subscribe.query_set_id, + rows: ws::QueryRows { + tables: vec![ws::SingleTableRows { + table: "connected".into(), + rows: BsatnRowList::new( + RowSizeHint::RowOffsets(vec![0].into()), + row.into(), + ), + }] + .into(), + }, + }), + ) + .await; + let _ = applied.send(generation); + } + ws::ClientMessage::CallReducer(_) => current.reducers += 1, + ws::ClientMessage::CallProcedure(_) => current.procedures += 1, + _ => panic!("unexpected fixture request"), + } + } + stats.push(current); + } + stats + }) + .await + .expect("owned WebSocket fixture exceeded deadline") + }); + (uri, rx, task) +} + +fn source() -> Identity { + Identity::from_hex(fixture::SOURCE).unwrap() +} +fn config(uri: &str, broker: &str) -> DbConnectionBuilder { + bindings::DbConnection::builder().with_container_credentials(Container::new(fixture::SOURCE, uri, broker).unwrap()) +} + +async fn drive_until(session: &mut ContainerSession, condition: impl Future) { + timeout(Duration::from_secs(5), async { + tokio::select! { + result = session.run() => panic!("session terminated unexpectedly: {result:?}"), + _ = condition => {}, + } + }) + .await + .unwrap(); +} + +#[test] +fn rejects_ambiguous_initial_configuration_without_io() { + let factory = |_, builder| builder; + assert!(matches!( + ContainerSession::new(bindings::DbConnection::builder(), factory), + Err(ContainerSessionError::Configuration) + )); + let new = || config("https://must-not-contact.invalid", "http://127.0.0.1:1/v1/credentials"); + for builder in [ + new().with_token(Some("owner-secret")), + new().with_debug_to_file("must-not-write"), + new().on_connect(|_, _, _| {}), + ] { + let error = ContainerSession::new(builder, factory).err().unwrap(); + assert_eq!(error, ContainerSessionError::Configuration); + assert!(!format!("{error:?} {error}").contains("owner-secret")); + } +} + +#[test] +fn requires_material_extension_of_both_returned_expiry_and_monotonic_deadline() { + let old = Validity { + expiry: UNIX_EPOCH + Duration::from_secs(100), + deadline: Instant::now(), + }; + assert!(!old.extended_by(Validity { + expiry: old.expiry, + deadline: old.deadline + Duration::from_secs(20) + })); + assert!(!old.extended_by(Validity { + expiry: old.expiry + Duration::from_secs(20), + deadline: old.deadline + })); + assert!(!old.extended_by(Validity { + expiry: old.expiry + Duration::from_millis(999), + deadline: old.deadline + Duration::from_secs(20) + })); + assert!(old.extended_by(Validity { + expiry: old.expiry + Duration::from_secs(2), + deadline: old.deadline + Duration::from_secs(2) + })); +} + +#[tokio::test] +async fn renewal_rebuilds_subscriptions_and_cache_without_replaying_calls() { + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("disposable-first", 3)), + fixture::response(200, &fixture::token_body("disposable-second", 20)), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _wire_applied, peer_task) = peer(2, source()).await; + let events = Arc::new(Mutex::new(Vec::new())); + let event_capture = events.clone(); + let (ready, mut readiness) = mpsc::unbounded_channel(); + let drops = Arc::new(AtomicUsize::new(0)); + let captures = drops.clone(); + let mut session = ContainerSession::new(config(&uri, &broker), move |info, builder| { + let ready = ready.clone(); + let captures = captures.clone(); + builder.on_connect(move |conn, _, _| { + // The first generation will receive a row before rotation. The + // second must still start with an empty, newly allocated cache. + assert_eq!(conn.db.connected().count(), 0); + conn.subscription_builder() + .on_applied(move |ctx| { + assert_eq!(ctx.db.connected().count(), 1); + ready.send(info.generation).unwrap(); + }) + .subscribe("SELECT * FROM connected"); + if info.generation == 1 { + struct Probe(Arc); + impl Drop for Probe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let probe = Probe(captures); + conn.reducers + .identity_connected_then(move |_, _| { + let _ = &probe; + panic!("unconfirmed reducer outcome must remain unknown"); + }) + .unwrap(); + } + }) + }) + .unwrap() + .on_event(move |event| event_capture.lock().unwrap().push(event)); + drive_until(&mut session, async { + assert_eq!(readiness.recv().await, Some(1)); + }) + .await; + let retained = session.current.as_ref().unwrap().context.clone(); + retained.invoke_procedure_with_callback::<_, ()>("unconfirmed", EmptyArgs {}, |_, _| { + panic!("unconfirmed procedure must not report completion") + }); + let first_aborts = session.current.as_ref().unwrap().aborts.clone(); + let first_cache = retained.cache.clone(); + drive_until(&mut session, async { + assert_eq!(readiness.recv().await, Some(2)); + }) + .await; + assert!(first_aborts.iter().all(AbortHandle::is_finished)); + assert!(!retained.is_active()); + assert!(!Arc::ptr_eq( + &first_cache, + &session.current.as_ref().unwrap().context.cache + )); + assert_eq!(drops.load(Ordering::SeqCst), 1); + session.shutdown_and_join().await.unwrap(); + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + let requests = broker_task.join().await; + assert_eq!(requests.len(), 2); + assert!(requests + .iter() + .all(|request| request.ends_with(&format!("{{\"target_database\":\"{}\"}}", fixture::SOURCE)))); + let stats = peer_task.join().await; + assert_eq!(stats[0].subscriptions, 1); + assert_eq!(stats[0].reducers, 1); + assert_eq!(stats[0].procedures, 1); + assert_eq!(stats[1].subscriptions, 1); + assert_eq!(stats[1].reducers, 0); + assert_eq!(stats[1].procedures, 0); + let events = events.lock().unwrap(); + let closed = events.iter().position(|event| matches!(event, ContainerSessionEvent::Closed { session, reason: ContainerSessionEndReason::CredentialRenewal, outstanding_calls: OutstandingCallOutcomes::Unknown } if session.generation == 1)).unwrap(); + let next = events + .iter() + .position(|event| matches!(event, ContainerSessionEvent::Connecting(info) if info.generation == 2)) + .unwrap(); + assert!(closed < next); + assert!(!format!("{events:?}").contains("disposable-")); +} + +#[tokio::test] +async fn unchanged_lease_expiry_does_not_rotate_and_denial_closes_the_active_session() { + let body = fixture::token_body("disposable-same-lease", 3); + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &body), + fixture::response(200, &body), + fixture::response(200, &body), + fixture::response(403, "private-error-body"), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let events = Arc::new(Mutex::new(Vec::new())); + let capture = events.clone(); + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(move |event| capture.lock().unwrap().push(event)); + let error = timeout(Duration::from_secs(5), session.run()) + .await + .unwrap() + .unwrap_err(); + assert_eq!( + error, + ContainerSessionError::Credential(ContainerCredentialError::Denied) + ); + assert!(session.current.is_none()); + assert_eq!(broker_task.join().await.len(), 4); + assert_eq!(peer_task.join().await.len(), 1); + assert_eq!( + events + .lock() + .unwrap() + .iter() + .filter(|event| matches!(event, ContainerSessionEvent::Connecting(_))) + .count(), + 1 + ); + assert!(!format!("{error:?} {:?}", events.lock().unwrap()).contains("private-error-body")); + assert_eq!(session.run().await.unwrap_err(), error); +} + +#[tokio::test] +async fn factory_cannot_override_the_pinned_target_or_supply_a_static_token() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-factory", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let mut session = ContainerSession::new(config("https://must-not-contact.invalid", &broker), |_, builder| { + builder.with_database_name("other").with_token(Some("owner-secret")) + }) + .unwrap(); + assert_eq!(session.run().await.unwrap_err(), ContainerSessionError::Configuration); + assert_eq!(broker_task.join().await.len(), 1); + assert!(session.current.is_none()); +} + +#[tokio::test] +async fn cancelling_token_request_preserves_resolved_target_and_closes_request_socket() { + let server = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}", server.local_addr().unwrap()); + let resolver = Task::spawn(async move { + let (mut socket, _) = server.accept().await.unwrap(); + assert!(fixture::read_request(&mut socket) + .await + .starts_with("GET /v1/database/alias/identity ")); + socket + .write_all(&fixture::response(200, fixture::SOURCE)) + .await + .unwrap(); + }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let (received, receipt) = oneshot::channel(); + let server = Task::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + received.send(()).unwrap(); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + let (mut socket, _) = listener.accept().await.unwrap(); + assert!(fixture::read_request(&mut socket).await.contains(fixture::SOURCE)); + socket.write_all(&fixture::response(403, "denied")).await.unwrap(); + }); + let mut session = + ContainerSession::new(config(&uri, &broker).with_database_name("alias"), |_, builder| builder).unwrap(); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + assert_eq!(session.target.as_ref().unwrap().1, source()); + resolver.join().await; + // Resolver is gone. Resume still requests exactly the pinned Identity. + assert_eq!( + session.run().await.unwrap_err(), + ContainerSessionError::Credential(ContainerCredentialError::Denied) + ); + server.join().await; + session.shutdown_and_join().await.unwrap_err(); +} + +#[tokio::test] +async fn dropping_owner_aborts_native_tasks_despite_retained_connection() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-drop", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let (connected, receipt) = oneshot::channel(); + let mut connected = Some(connected); + let mut session = ContainerSession::new(config(&uri, &broker), move |_, builder| { + let connected = connected.take().unwrap(); + builder.on_connect(move |_, _, _| { + connected.send(()).unwrap(); + }) + }) + .unwrap(); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + let retained = session.current.as_ref().unwrap().context.clone(); + let tasks = retained.inner.lock().unwrap().background_tasks.clone(); + drop(session); + assert!(!retained.is_active()); + tasks.lock().await.stop_and_join().await.unwrap(); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn cancelling_shutdown_keeps_native_join_ownership_until_resumed() { + // A task destructor deliberately waits on a gate. This proves that the + // shutdown method retains the JoinHandle while its wait is cancelled. + let (entered, entry) = oneshot::channel(); + let (release, gate) = std::sync::mpsc::channel(); + struct BlockingDrop { + entered: Option>, + gate: std::sync::mpsc::Receiver<()>, + } + impl Drop for BlockingDrop { + fn drop(&mut self) { + let _ = self.entered.take().unwrap().send(()); + self.gate.recv_timeout(Duration::from_secs(3)).unwrap(); + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let destructor = BlockingDrop { + entered: Some(entered), + gate, + }; + let websocket = runtime.spawn(async move { + let _destructor = destructor; + futures::future::pending::<()>().await; + }); + let parser = runtime.spawn(futures::future::pending()); + let mut session = ContainerSession::new( + config("http://127.0.0.1:1", "http://127.0.0.1:1/v1/credentials"), + |_, builder| builder, + ) + .unwrap(); + let inner = + super::super::build_db_ctx_inner::(None, NativeTasks::new(websocket, parser), None, None, None); + let aborts = inner + .lock() + .unwrap() + .background_tasks + .try_lock() + .unwrap() + .abort_handles(); + let (outgoing, _outgoing_rx) = futures_channel::mpsc::unbounded(); + let (_incoming, incoming_rx) = futures_channel::mpsc::unbounded(); + let (pending, pending_rx) = futures_channel::mpsc::unbounded(); + let context = super::super::build_db_ctx( + tokio::runtime::Handle::current(), + inner, + outgoing, + Arc::new(tokio::sync::Mutex::new(incoming_rx)), + pending, + Arc::new(tokio::sync::Mutex::new(pending_rx)), + None, + None, + ); + session.current = Some(Current { + context, + info: ContainerSessionInfo { + generation: 1, + target: source(), + }, + validity: Validity { + expiry: SystemTime::now(), + deadline: Instant::now(), + }, + refresh_at: Instant::now(), + signal: Arc::new(Signal::default()), + connected_announced: false, + closing: None, + driver_finished: false, + aborts, + }); + { + let mut shutdown = Box::pin(session.shutdown_and_join()); + tokio::select! { result = &mut shutdown => panic!("shutdown completed before destructor release: {result:?}"), result = entry => result.unwrap(), } + } + assert!(session.current.is_some()); + release.send(()).unwrap(); + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + session.shutdown_and_join().await.unwrap(); + runtime.shutdown_background(); +} + +#[tokio::test] +async fn callback_panic_propagates_only_after_socket_and_native_tasks_close() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-panic", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("injected application panic")) + }) + .unwrap(); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert!(session.current.is_none()); + assert!(session.stopped); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn observer_panic_during_rotation_still_joins_the_previous_connection() { + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("disposable-old", 3)), + fixture::response(200, &fixture::token_body("disposable-new", 20)), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(|event| { + if matches!(event, ContainerSessionEvent::Closing { .. }) { + panic!("injected observer panic"); + } + }); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn unexpected_sender_is_not_exposed_to_application_callback() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-identity", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, Identity::ZERO).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("wrong sender must never reach application")) + }) + .unwrap(); + assert_eq!( + session.run().await.unwrap_err(), + ContainerSessionError::IdentityMismatch + ); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn stalled_refresh_is_cancelled_at_expiry_and_never_extends_the_old_connection() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let broker_task = Task::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + socket + .write_all(&fixture::response(200, &fixture::token_body("disposable-expiring", 2))) + .await + .unwrap(); + drop(socket); + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(3), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let (uri, _, peer_task) = peer(1, source()).await; + let (closed, receipt) = oneshot::channel(); + let mut closed = Some(closed); + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(move |event| { + if matches!( + event, + ContainerSessionEvent::Closing { + reason: ContainerSessionEndReason::CredentialExpiry, + .. + } + ) { + closed.take().unwrap().send(()).unwrap(); + } + }); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + if let Some(current) = &session.current { + assert!(!current.context.is_active()); + } + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn factory_panic_leaves_no_connection_and_is_a_terminal_owner_failure() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-factory-panic", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let mut session = ContainerSession::new(config("https://must-not-contact.invalid", &broker), |_, _| { + panic!("injected factory panic") + }) + .unwrap(); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert_eq!(session.run().await.unwrap_err(), ContainerSessionError::Panicked); + assert!(session.current.is_none()); + broker_task.join().await; +} + +#[tokio::test] +async fn unexpected_sender_still_fails_when_initial_connection_is_followed_by_immediate_close() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-fast-close", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer_with_close(1, Identity::ZERO, true).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("wrong sender must never reach application")) + }) + .unwrap(); + assert_eq!( + timeout(Duration::from_secs(3), session.run()) + .await + .unwrap() + .unwrap_err(), + ContainerSessionError::IdentityMismatch + ); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} diff --git a/sdks/rust/src/db_connection/native_tasks.rs b/sdks/rust/src/db_connection/native_tasks.rs new file mode 100644 index 00000000000..bf27f3e4a2e --- /dev/null +++ b/sdks/rust/src/db_connection/native_tasks.rs @@ -0,0 +1,80 @@ +//! Native connection tasks retained through terminal cleanup. + +use crate::__codegen::InternalError; +use tokio::task::JoinHandle; + +#[cfg(test)] +mod tests; + +#[derive(Default)] +pub(super) struct NativeTasks { + websocket: Option>, + parser: Option>, + failure: Option, +} + +impl NativeTasks { + pub fn new(websocket: JoinHandle<()>, parser: JoinHandle<()>) -> Self { + Self { + websocket: Some(websocket), + parser: Some(parser), + failure: None, + } + } + + pub fn abort_handles(&self) -> Vec { + [&self.websocket, &self.parser] + .into_iter() + .flatten() + .map(JoinHandle::abort_handle) + .collect() + } + + pub fn record_failure(&mut self, error: crate::Error) { + // A cancelled terminal wait must not discard its original failure. + self.failure.get_or_insert(error); + } + + /// Called only after event processing has terminated. In particular, a + /// parser error must not leave the WebSocket waiting for more input. + pub async fn stop_and_join(&mut self) -> crate::Result<()> { + self.abort(); + // Each handle stays in shared connection state across await. Clear it + // immediately after joining, before the next suspension point, so a + // cancelled caller cannot detach it or poll a completed handle twice. + Self::join(&mut self.websocket, &mut self.failure, "WebSocket").await; + Self::join(&mut self.parser, &mut self.failure, "parser").await; + self.failure.clone().map_or(Ok(()), Err) + } + + async fn join(handle: &mut Option>, failure: &mut Option, name: &str) { + let Some(task) = handle.as_mut() else { return }; + let result = task.await; + *handle = None; + if let Err(error) = result { + // Cancellation was requested above. A task panic is still a failure, + // and repeated terminal waits must continue reporting that failure. + if !error.is_cancelled() && failure.is_none() { + *failure = Some( + InternalError::new(format!("Native {name} task failed")) + .with_cause(error) + .into(), + ); + } + } + } + + fn abort(&self) { + for task in [&self.websocket, &self.parser].into_iter().flatten() { + task.abort(); + } + } +} + +impl Drop for NativeTasks { + fn drop(&mut self) { + // Dropping the last connection is a cancellation request, not a join. + // Owners requiring completed shutdown must finish run_async/run_threaded. + self.abort(); + } +} diff --git a/sdks/rust/src/db_connection/native_tasks/tests.rs b/sdks/rust/src/db_connection/native_tasks/tests.rs new file mode 100644 index 00000000000..1a3906055d6 --- /dev/null +++ b/sdks/rust/src/db_connection/native_tasks/tests.rs @@ -0,0 +1,210 @@ +use super::*; +use crate::db_connection::{ + build_db_ctx, build_db_ctx_inner, terminal_tests::bindings::RemoteModule, DbConnectionBuilder, ParsedMessage, +}; +use futures::{FutureExt, SinkExt, StreamExt}; +use spacetimedb_client_api_messages::websocket::v2 as ws; +use spacetimedb_lib::{bsatn, ConnectionId, Identity, Timestamp}; +use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use tokio::{net::TcpListener, sync::oneshot}; +use tokio_tungstenite::tungstenite::{handshake::server::Response, Message}; + +async fn blocked_task() -> (JoinHandle<()>, std::sync::mpsc::Sender<()>) { + let (started, ready) = oneshot::channel(); + let (release, blocked) = std::sync::mpsc::channel(); + let task = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + // Dropping release on a failed assertion also releases this worker. + let _ = blocked.recv(); + }); + ready.await.unwrap(); + (task, release) +} + +#[tokio::test] +async fn cancellation_during_second_join_preserves_owner_for_retry() { + let first = tokio::spawn(std::future::pending()); + let (second, release) = blocked_task().await; + let mut tasks = NativeTasks::new(first, second); + assert!(tokio::time::timeout(Duration::from_millis(30), tasks.stop_and_join()) + .await + .is_err()); + assert!(tasks.websocket.is_none(), "first task should already have been joined"); + assert!(tasks.parser.is_some(), "pending task must still have an owner"); + release.send(()).unwrap(); + tasks.stop_and_join().await.unwrap(); + assert!(tasks.parser.is_none()); + tasks.stop_and_join().await.unwrap(); +} + +#[tokio::test] +async fn task_panic_still_joins_other_task_and_remains_an_error_after_retry() { + let (panicking, ready) = oneshot::channel(); + let first = tokio::spawn(async move { + let _ = panicking.send(()); + panic!("injected connection task panic"); + }); + ready.await.unwrap(); + let (second, release) = blocked_task().await; + let mut tasks = NativeTasks::new(first, second); + assert!(tokio::time::timeout(Duration::from_millis(30), tasks.stop_and_join()) + .await + .is_err()); + assert!(tasks.failure.is_some()); + assert!(tasks.websocket.is_none()); + assert!(tasks.parser.is_some()); + release.send(()).unwrap(); + assert!(tasks.stop_and_join().await.is_err()); + assert!(tasks.parser.is_none()); + assert!(tasks.stop_and_join().await.is_err()); +} + +#[tokio::test] +async fn shutdown_drops_both_task_resources_before_returning() { + let resource = Arc::new(()); + let first_capture = resource.clone(); + let second_capture = resource.clone(); + let first = tokio::spawn(async move { + let _capture = first_capture; + std::future::pending::<()>().await; + }); + let second = tokio::spawn(async move { + let _capture = second_capture; + std::future::pending::<()>().await; + }); + let mut tasks = NativeTasks::new(first, second); + tasks.stop_and_join().await.unwrap(); + assert_eq!(Arc::strong_count(&resource), 1); +} + +#[tokio::test] +async fn cancelled_terminal_driver_preserves_original_processing_error() { + let first = tokio::spawn(std::future::pending()); + let (second, release) = blocked_task().await; + let inner = build_db_ctx_inner::(None, NativeTasks::new(first, second), None, None, None); + let (outgoing, _outgoing_recv) = futures_channel::mpsc::unbounded(); + let (incoming, incoming_recv) = futures_channel::mpsc::unbounded(); + let (pending, pending_recv) = futures_channel::mpsc::unbounded(); + incoming + .unbounded_send(ParsedMessage::Error( + InternalError::new("original processing failure").into(), + )) + .unwrap(); + drop(incoming); + let context = build_db_ctx( + tokio::runtime::Handle::current(), + inner, + outgoing, + Arc::new(tokio::sync::Mutex::new(incoming_recv)), + pending, + Arc::new(tokio::sync::Mutex::new(pending_recv)), + None, + None, + ); + assert!(tokio::time::timeout(Duration::from_millis(30), context.run_async()) + .await + .is_err()); + release.send(()).unwrap(); + let error = context.run_async().await.unwrap_err(); + assert!(error.to_string().contains("original processing failure")); +} + +async fn terminal_connection_joins_tasks(callback_panics: bool, poison_sender: bool) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (closed, closure) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (socket, address) = listener.accept().await.unwrap(); + assert!(address.ip().is_loopback()); + let mut socket = + tokio_tungstenite::accept_hdr_async(socket, |_: &http::Request<()>, mut response: Response| { + response + .headers_mut() + .insert("sec-websocket-protocol", ws::BIN_PROTOCOL.parse().unwrap()); + Ok(response) + }) + .await + .unwrap(); + let message = if callback_panics { + ws::ServerMessage::InitialConnection(ws::InitialConnection { + identity: Identity::from_u256(1u32.into()), + connection_id: ConnectionId::from_u128(1), + token: "disposable-test-token".into(), + }) + } else { + // A well-formed response for a request this client never made is a + // processing error. The peer deliberately keeps its socket open. + ws::ServerMessage::ReducerResult(ws::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: ws::ReducerOutcome::OkEmpty, + }) + }; + let mut bytes = vec![0]; // No compression. + bytes.extend(bsatn::to_vec(&message).unwrap()); + socket.send(Message::Binary(bytes.into())).await.unwrap(); + while let Some(message) = socket.next().await { + if message.is_err() || matches!(message, Ok(Message::Close(_))) { + break; + } + } + let _ = closed.send(()); + }); + // Keep a positive join even when the test body fails or times out. + let result = AssertUnwindSafe(tokio::time::timeout(Duration::from_secs(3), async { + let context = DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-native-task-test") + .on_connect(move |_, _, _| { + assert!(!callback_panics, "injected application callback panic"); + }) + .build_native_impl(tokio::runtime::Handle::current()) + .await + .unwrap(); + if poison_sender { + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _outgoing = context.send_chan.lock().unwrap(); + panic!("injected outgoing queue panic while holding its mutex"); + })); + assert!(result.is_err()); + } + let result = AssertUnwindSafe(context.run_async()).catch_unwind().await; + if callback_panics || poison_sender { + assert!(result.is_err(), "callback panic should propagate after cleanup"); + } else { + assert!(result.unwrap().is_err(), "unsolicited reducer result must fail"); + } + assert!(context + .send_chan + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_none()); + let tasks = context.inner.lock().unwrap().background_tasks.clone(); + let tasks = tasks.lock().await; + assert!(tasks.websocket.is_none() && tasks.parser.is_none()); + closure.await.expect("peer disappeared before observing socket closure"); + })) + .catch_unwind() + .await; + peer.abort(); + let joined = peer.await; + if let Err(error) = joined { + assert!(error.is_cancelled(), "test peer failed: {error}"); + } + result.unwrap().expect("connection did not finish native task cleanup"); +} + +#[tokio::test] +async fn processing_error_closes_socket_and_joins_tasks_with_retained_connection() { + terminal_connection_joins_tasks(false, false).await; +} + +#[tokio::test] +async fn callback_panic_joins_tasks_before_propagating_unwind() { + terminal_connection_joins_tasks(true, false).await; +} + +#[tokio::test] +async fn poisoned_outgoing_mutex_does_not_bypass_native_task_join() { + terminal_connection_joins_tasks(false, true).await; +} diff --git a/sdks/rust/src/db_connection/terminal_tests.rs b/sdks/rust/src/db_connection/terminal_tests.rs new file mode 100644 index 00000000000..b88392dfb3a --- /dev/null +++ b/sdks/rust/src/db_connection/terminal_tests.rs @@ -0,0 +1,251 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[path = "../../tests/connect_disconnect_client/src/module_bindings/mod.rs"] +pub(super) mod bindings; +use bindings::RemoteModule; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct Args {} +impl InModule for Args { + type Module = RemoteModule; +} +impl From for bindings::Reducer { + fn from(_: Args) -> Self { + Self::IdentityConnected + } +} + +struct DropProbe { + context: DbContextImpl, + drops: Arc, +} +impl Drop for DropProbe { + fn drop(&mut self) { + assert!( + self.context.inner.try_lock().is_ok(), + "capture dropped under inner lock" + ); + assert!( + self.context.send_chan.try_lock().is_ok(), + "capture dropped under send lock" + ); + assert!( + self.context.pending_mutations_recv.try_lock().is_ok(), + "capture dropped under queue lock" + ); + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +fn fixture( + runtime: &Runtime, + disconnects: Arc, +) -> ( + DbContextImpl, + mpsc::UnboundedSender>, +) { + let inner = build_db_ctx_inner::( + None, + NativeTasks::default(), + None, + None, + Some(Box::new(move |_, error| { + assert!(error.is_none()); + disconnects.fetch_add(1, Ordering::SeqCst); + })), + ); + inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connected; + let (outgoing, outgoing_recv) = mpsc::unbounded(); + // Keep the transport receiver alive without any network or server fixture. + let (incoming, incoming_recv) = mpsc::unbounded(); + let (pending, pending_recv) = mpsc::unbounded(); + let context = build_db_ctx( + runtime.handle().clone(), + inner, + outgoing, + Arc::new(TokioMutex::new(incoming_recv)), + pending, + Arc::new(TokioMutex::new(pending_recv)), + Some(ConnectionId::from_u128(1)), + None, + ); + runtime.spawn(async move { + let mut outgoing_recv = outgoing_recv; + while outgoing_recv.next().await.is_some() {} + }); + (context, incoming) +} + +fn queue_calls(context: &DbContextImpl, drops: &Arc) { + let reducer = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context + .invoke_reducer_with_callback(Args {}, move |_, _| { + let _capture = reducer; + panic!("unknown-outcome reducer must not report completion"); + }) + .unwrap(); + let procedure = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| { + let _capture = procedure; + panic!("unknown-outcome procedure must not report completion"); + }); +} + +#[test] +fn terminal_disconnect_releases_inflight_and_queued_requests_with_retained_context() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + queue_calls(&context, &drops); + assert_eq!(drops.load(Ordering::SeqCst), 0); + assert!(matches!(context.end_connection(None), crate::Error::Disconnected)); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); + assert!(!context.is_active()); + + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + assert!(matches!( + context.invoke_reducer_with_callback(Args {}, move |_, _| drop(late)), + Err(crate::Error::Disconnected) + )); + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| drop(late)); + assert_eq!(drops.load(Ordering::SeqCst), 6); + context.end_connection(None); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn queued_disconnect_releases_calls_and_drains_late_results_until_terminal_event() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + queue_calls(&context, &drops); + incoming + .unbounded_send(ParsedMessage::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: Ok(Ok(bindings::DbUpdate::default())), + }) + .unwrap(); + incoming + .unbounded_send(ParsedMessage::ProcedureResult { + request_id: u32::MAX, + result: Ok(Bytes::new()), + }) + .unwrap(); + context.frame_tick().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!( + disconnects.load(Ordering::SeqCst), + 0, + "local request is not the terminal callback" + ); + assert!(!context.is_active()); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn retained_table_and_subscription_handles_cannot_retain_callbacks_after_terminal() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects); + let table = context.get_table::("connected"); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let registered = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let queued = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.end_connection(None); + assert_eq!(drops.load(Ordering::SeqCst), 3); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(®istered)); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&queued)); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + assert_eq!(drops.load(Ordering::SeqCst), 5); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); + assert_eq!(table.iter().count(), 0); +} + +#[test] +fn cancelling_before_initial_message_releases_registered_subscription_without_callbacks() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + context.inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connecting; + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + context.frame_tick().unwrap(); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); +} diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index e7efd286514..6d1decc56ae 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -10,6 +10,9 @@ // code generated by the CLI's codegen references them, // but users should not. +#[cfg(test)] +extern crate self as spacetimedb_sdk; + mod callbacks; mod client_cache; mod compression; @@ -25,6 +28,11 @@ pub mod error; pub mod event; pub mod table; +#[cfg(not(feature = "browser"))] +pub use db_connection::container_session::{ + ContainerSession, ContainerSessionEndReason, ContainerSessionError, ContainerSessionEvent, ContainerSessionInfo, + ContainerSessionRetryReason, OutstandingCallOutcomes, +}; pub use db_connection::DbConnectionBuilder; pub use db_context::DbContext; pub use error::{Error, Result}; diff --git a/sdks/rust/src/subscription.rs b/sdks/rust/src/subscription.rs index 2204985ea15..3fb4be3b5b2 100644 --- a/sdks/rust/src/subscription.rs +++ b/sdks/rust/src/subscription.rs @@ -47,23 +47,11 @@ pub(crate) enum PendingUnsubscribeResult { impl SubscriptionManager { pub(crate) fn on_disconnect(&mut self, _ctx: &M::ErrorContext) { - // We need to clear all the subscriptions. - // TODO: is this correct? We don't remove them from the client cache, - // we may want to resume them in the future if we impl reconnecting, - // and users can already register on-disconnect callbacks which will run in this case. - - // NOTE(cloutiertyler) - // This function previously invoke `on_error` for all subscriptions. - // However, this is inconsistent behavior given that `on_disconnect` for - // connections no longer always has an error argument and that the user - // can add an `on_ended` callback when unsubscribing. - // - // We propose instead that `on_ended` be added to the subscription - // builder so that it can be invoked when the subscription is ended - // because of a normal disconnect, but without the user calling - // `unsubscribe_then`. This can be done in a non-breaking way. - // - // For now, we will just do nothing when a subscription ends normally. + // Disconnect does not synthesize on_error/on_ended callbacks. Retained + // handles must nevertheless stop retaining callback captures. + for (_, handle) in std::mem::take(&mut self.subscriptions) { + handle.cancel_pending_callbacks(); + } } /// Register a new subscription. This does not send the subscription to the server. @@ -187,13 +175,17 @@ impl SubscriptionBuilder { self.on_applied, self.on_error, )); - self.conn + if self + .conn .pending_mutations_send .unbounded_send(PendingMutation::Subscribe { query_set_id, handle: handle.clone(), }) - .unwrap(); + .is_err() + { + handle.cancel_pending_callbacks(); + } M::SubscriptionHandle::new(handle) } @@ -471,6 +463,16 @@ impl SubscriptionHandleImpl { } } + pub(crate) fn cancel_pending_callbacks(&self) { + let callbacks = { + let mut inner = self.inner.lock().unwrap(); + inner.status = SubscriptionServerState::Ended; + (inner.on_applied.take(), inner.on_error.take(), inner.on_ended.take()) + }; + // Callback destructors may inspect or unsubscribe this same handle. + drop(callbacks); + } + pub(crate) fn start(&self) -> Option { let mut inner = self.inner.lock().unwrap(); inner.start() diff --git a/tools/ci/windows-exec-tests.ps1 b/tools/ci/windows-exec-tests.ps1 new file mode 100644 index 00000000000..eb561733310 --- /dev/null +++ b/tools/ci/windows-exec-tests.ps1 @@ -0,0 +1,192 @@ +# Compile the actual CLI libtest, then run only owned local exec fixtures. +# A forced cleanup is a failed gate, never evidence that the fixture joined. +# These receipts cover direct children. Each native fixture owns and joins its +# own children; the wrapper does not claim independent descendant retirement. +$ErrorActionPreference = 'Stop' +if (-not $IsWindows -or $env:RUNNER_OS -ne 'Windows') { + throw 'This gate requires the native Windows CI runner.' +} + +$workspace = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +$results = Join-Path $env:RUNNER_TEMP ('container-exec-' + [Guid]::NewGuid().ToString('N')) +if (Test-Path $results) { throw 'The owned result directory already exists.' } +$null = New-Item -ItemType Directory -Path $results +$runtime = Join-Path $results 'runtime' +$null = New-Item -ItemType Directory -Path $runtime +"WINDOWS_EXEC_RESULTS=$results" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +$windows = [Environment]::GetFolderPath([Environment+SpecialFolder]::Windows) +if (-not [IO.Path]::IsPathFullyQualified($windows) -or -not (Test-Path $windows -PathType Container)) { + throw 'The Windows system directory is unavailable.' +} +$system32 = Join-Path $windows 'System32' +$report = [ordered]@{ + state = 'running' + windows = [Environment]::OSVersion.VersionString + processes = [Collections.Generic.List[object]]::new() +} + +function Invoke-OwnedProcess { + param([string]$Executable, [string[]]$Arguments, [string]$Name, + [int]$TimeoutSeconds, [switch]$Sterile) + $stdoutPath = Join-Path $results "$Name.stdout" + $stderrPath = Join-Path $results "$Name.stderr" + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $Executable + $start.UseShellExecute = $false + $start.WorkingDirectory = if ($Sterile) { $runtime } else { $workspace } + $start.RedirectStandardInput = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { $start.ArgumentList.Add($argument) } + if ($Sterile) { + $start.Environment.Clear() + $start.Environment['SystemRoot'] = $windows + $start.Environment['WINDIR'] = $windows + $start.Environment['PATH'] = $system32 + $start.Environment['COMSPEC'] = Join-Path $system32 'cmd.exe' + $start.Environment['TEMP'] = $runtime + $start.Environment['TMP'] = $runtime + $start.Environment['RUST_BACKTRACE'] = '1' + } + $process = [Diagnostics.Process]::new() + $process.StartInfo = $start + $stdout = [IO.File]::Open($stdoutPath, 'CreateNew', 'Write', 'ReadWrite') + $stderr = [IO.File]::Open($stderrPath, 'CreateNew', 'Write', 'ReadWrite') + $cancel = [Threading.CancellationTokenSource]::new() + $receipt = [ordered]@{ + name = $Name; started = $false; waited = $false; forced = $false; streamsJoined = $false + cleanupErrors = [Collections.Generic.List[string]]::new() + } + $report.processes.Add($receipt) + $copies = @() + $failure = $null + $clock = [Diagnostics.Stopwatch]::StartNew() + try { + if (-not $process.Start()) { throw 'The owned child did not start.' } + $receipt.started = $true + $receipt.pid = $process.Id + $process.StandardInput.Close() + $copies = @( + $process.StandardOutput.BaseStream.CopyToAsync($stdout, 81920, $cancel.Token), + $process.StandardError.BaseStream.CopyToAsync($stderr, 81920, $cancel.Token) + ) + while (-not $process.WaitForExit(100)) { + if ($clock.Elapsed.TotalSeconds -ge $TimeoutSeconds) { throw 'The owned child exceeded its time bound.' } + if ($stdout.Length + $stderr.Length -gt 64MB) { throw 'The owned child exceeded its output bound.' } + } + $receipt.waited = $true + $receipt.exitCode = $process.ExitCode + if (-not [Threading.Tasks.Task]::WhenAll([Threading.Tasks.Task[]]$copies).Wait(5000)) { + throw 'The exited child retained output handles.' + } + $receipt.streamsJoined = $true + if ($stdout.Length + $stderr.Length -gt 64MB) { throw 'The owned child exceeded its output bound.' } + if ($process.ExitCode -ne 0) { throw "The $Name child failed; see retained output." } + } catch { + $failure = $_ + $receipt.failure = $_.Exception.Message + } finally { + try { + try { + if ($receipt.started -and -not $process.HasExited) { + $receipt.forced = $true + $process.Kill($true) + } + } catch { + $receipt.cleanupErrors.Add('Terminate direct child/tree: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + try { + if ($receipt.started) { + if (-not $process.WaitForExit(30000)) { throw 'The direct child did not terminate.' } + $receipt.waited = $true + $receipt.exitCode = $process.ExitCode + } + } catch { + $receipt.cleanupErrors.Add('Join direct child: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + try { + if (-not $receipt.streamsJoined) { + $cancel.Cancel() + if ($receipt.started) { + try { $process.StandardOutput.Close() } finally { $process.StandardError.Close() } + } + foreach ($copy in $copies) { + try { $copy.GetAwaiter().GetResult() } catch { } + } + $receipt.streamsJoined = $true + } + } catch { + $receipt.cleanupErrors.Add('Join output readers: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + } finally { + $receipt.seconds = $clock.Elapsed.TotalSeconds + $stdout.Dispose() + $stderr.Dispose() + $cancel.Dispose() + $process.Dispose() + } + } + if ($null -ne $failure) { throw $failure } + return [IO.File]::ReadAllText($stdoutPath) +} + +try { + $cargo = (Get-Command cargo -CommandType Application).Source + $rustc = (Get-Command rustc -CommandType Application).Source + $report.rust = Invoke-OwnedProcess $rustc @('--version', '--verbose') 'rust-version' 30 + $toolchain = Get-Content -Raw (Join-Path $workspace 'rust-toolchain.toml') + $channel = [regex]::Match($toolchain, '(?m)^channel = "([0-9]+\.[0-9]+\.[0-9]+)"\r?$') + if (-not $channel.Success -or + $report.rust -notmatch ('(?m)^release: ' + [regex]::Escape($channel.Groups[1].Value) + '\r?$') -or + $report.rust -notmatch '(?m)^host: x86_64-pc-windows-msvc\r?$') { + throw 'This gate requires the repository MSVC toolchain.' + } + $build = Invoke-OwnedProcess $cargo @('test', '--locked', '--release', '-p', 'spacetimedb-cli', '--lib', '--no-run', '--message-format=json') 'compile' 600 + $artifacts = @($build -split '\r?\n' | Where-Object { $_ } | ForEach-Object { + $message = $_ | ConvertFrom-Json + if ($message.reason -eq 'compiler-artifact' -and $message.profile.test -eq $true -and + $message.target.name -eq 'spacetimedb_cli' -and $message.target.kind -contains 'lib' -and $message.executable) { + $message.executable + } + } | Select-Object -Unique) + if ($artifacts.Count -ne 1) { throw 'Expected exactly one actual CLI libtest artifact.' } + $binary = (Resolve-Path $artifacts[0]).Path + $report.executable = $binary + $report.executableSha256 = (Get-FileHash -Algorithm SHA256 $binary).Hash.ToLowerInvariant() + $prefix = 'subcommands::container::execute::' + $required = @( + 'native_windows_pipe_backpressure_cancellation_joins_both_workers', + 'native_windows_overlapped_pipe_cancel_observes_exact_completion', + 'native_windows_overlapped_pipe_suppresses_inherited_completion_port_packets', + 'native_windows_close_before_ready_restores_then_releases_callback', + 'native_windows_conpty_normal_error_cancel_restore_console', + 'native_windows_console_cleanup_reports_drain_failure' + ) + $listing = Invoke-OwnedProcess $binary @($prefix, '--list') 'listing' 30 -Sterile + $selected = @() + foreach ($suffix in $required) { + $matches = [regex]::Matches($listing, '(?m)^(' + [regex]::Escape($prefix) + '[^\r\n]*::' + [regex]::Escape($suffix) + '): test\r?$') + if ($matches.Count -ne 1) { throw "Missing or ambiguous required native test: $suffix" } + $selected += $matches[0].Groups[1].Value + } + $report.requiredNativeTests = $selected + $output = Invoke-OwnedProcess $binary @($prefix, '--test-threads=1', '--show-output', '--format=pretty') 'tests' 240 -Sterile + foreach ($name in $selected) { + if (-not [regex]::IsMatch($output, '(?m)^test ' + [regex]::Escape($name) + ' \.\.\. ok\r?$')) { + throw "Required native test did not pass: $name" + } + } + if (-not [regex]::IsMatch($output, '(?m)^test result: ok\. [1-9][0-9]* passed; 0 failed;')) { + throw 'The exec suite did not report a nonempty passing result.' + } + $report.state = 'passed' +} catch { + $report.state = 'failed' + $report.failure = $_.Exception.Message + throw +} finally { + $report | ConvertTo-Json -Depth 8 | Set-Content -Encoding utf8 (Join-Path $results 'report.json') +}