diff --git a/.github/workflows/gateway-cluster-tests.yml b/.github/workflows/gateway-cluster-tests.yml new file mode 100644 index 000000000..b89002250 --- /dev/null +++ b/.github/workflows/gateway-cluster-tests.yml @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway cluster tests + +# WaveKV replication between gateway nodes: push and periodic sync, anti-entropy +# repair, bootstrap after losing a store, partition recovery, node identity, and +# the admin RPCs that gate registration. Unit tests cover the store; this runs +# three real gateways and stops, wipes and restarts them. +# +# The nodes authenticate each other for real, so the cluster mTLS path is +# exercised rather than switched off -- which is what the process-based suite +# this replaces could not do. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-cluster-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-cluster: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # 28 tests, most of which restart nodes and then wait out a 5s sync interval. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-cluster-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-cluster-${{ runner.os }}- + + - name: Cluster suite + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh + + # The suite only materialises node logs for the two tests that assert on + # them, so the mounted directory is close to empty for any other failure. + # Pull them from the daemon instead, by compose service so the shared + # fixture's project-scoped names do not have to be guessed. + - name: Collect node logs on failure + if: failure() + working-directory: dstack/gateway/test-run/cluster + run: | + mkdir -p run/logs + # By project, discovered from the daemon. A bare `docker compose` here + # addresses the project named after this directory (`cluster`), which + # no test ever uses -- every test runs under `cluster-`, so + # the loop that used `compose config --services` collected nothing. + docker ps -a --filter 'label=com.docker.compose.project' \ + --format '{{.Label "com.docker.compose.project"}}' \ + | grep -E '^cluster-' | sort -u | while read -r project; do + docker compose -p "$project" -f docker-compose.yml logs --no-color \ + > "run/logs/project-$project.log" 2>&1 || true + done + # The suite's own per-test dumps are root-owned inside the bind mount. + docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + FIXTURE_NS=dstack-fixture-cluster docker compose -p dstack-fixture-cluster -f ../attestation/fixture.yml logs \ + --no-color > "run/logs/fixture.log" 2>&1 || true + docker compose ps -a > run/logs/compose-ps.txt 2>&1 || true + + - name: Upload node logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-cluster-logs + path: dstack/gateway/test-run/cluster/run/logs/ + if-no-files-found: ignore + retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh down diff --git a/.github/workflows/gateway-e2e-tests.yml b/.github/workflows/gateway-e2e-tests.yml new file mode 100644 index 000000000..359f58a1f --- /dev/null +++ b/.github/workflows/gateway-e2e-tests.yml @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway e2e tests + +# The gateway's certbot/ACME half only exists as a whole: certificate issuance, +# cluster sync of the result, and the dns-persist-01 flow each depend on the +# ones before. Unit tests cover the pieces; this stands up three gateways, a +# Pebble CA, a mock Cloudflare DNS API and a mock attestation collateral +# service, and asserts on what the cluster actually converges to. +# +# The suite verifies peer quotes for real: the gateways check each other against +# development trust anchors derived from the simulator's seed, so it covers the +# cluster mTLS path that a harness with the checks switched off cannot reach. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'sdk/simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-e2e-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-e2e: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # A cold musl build dominates; the suite itself waits out real ACME orders + # and a 20s cluster-sync settle, so it is minutes rather than seconds. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + # run-e2e.sh builds a static gateway so the image can be alpine. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers, + # so the host kernel has to offer the module. Record it: a run that failed + # because the runner image changed under us should say so plainly instead + # of looking like a gateway bug. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + docker version --format 'docker: {{.Server.Version}}' + docker compose version + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-e2e-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-e2e-${{ runner.os }}- + + # --keep-running so the containers survive for the log step below; the + # script's own EXIT trap would otherwise tear them down and leave nothing + # to collect from exactly the runs worth diagnosing. + - name: Gateway e2e suite + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh --keep-running + + - name: Collect container logs on failure + if: failure() + working-directory: dstack/gateway/test-run/e2e + run: | + # By compose service, not by container name: the shared attestation + # fixture deliberately does not pin one, so that more than one suite + # can be up at a time. + mkdir -p /tmp/gateway-e2e-logs + for svc in $(docker compose config --services); do + docker compose logs --no-color "$svc" > "/tmp/gateway-e2e-logs/$svc.log" 2>&1 || true + done + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + FIXTURE_NS=dstack-fixture-e2e docker compose -p dstack-fixture-e2e -f ../attestation/fixture.yml logs \ + --no-color > "/tmp/gateway-e2e-logs/fixture.log" 2>&1 || true + docker compose ps -a > /tmp/gateway-e2e-logs/compose-ps.txt 2>&1 || true + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh down + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-e2e-logs + path: /tmp/gateway-e2e-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/gateway-proxy-tests.yml b/.github/workflows/gateway-proxy-tests.yml index 48b56e528..86413a4c1 100644 --- a/.github/workflows/gateway-proxy-tests.yml +++ b/.github/workflows/gateway-proxy-tests.yml @@ -8,20 +8,53 @@ name: Gateway proxy tests # `ktls`) whose behaviour depends on kernel capabilities and on a per-connection # gate. Unit tests cover the relay functions; this runs a real gateway process # and asserts on what actually reaches the wire. +# +# The suite runs in containers. One of them has a seccomp profile that makes +# `setsockopt(IPPROTO_TCP, TCP_ULP)` fail, which is how the no-TLS-ULP fallback +# is exercised now -- the suite used to `rmmod tls`, which needed passwordless +# sudo, took the module from the whole host, and skipped itself whenever +# anything else was using TLS. on: push: branches: [ next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/vendor/ktls/**' + # The suite runs the gateway with attestation on, so it depends on the + # shared fixture: the simulator, the mock collateral service, and the + # fixture payloads baked into the simulator image. + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'sdk/simulator/**' - '.github/workflows/gateway-proxy-tests.yml' pull_request: branches: [ next, 'release/**' ] paths: - 'dstack/gateway/**' - 'dstack/vendor/ktls/**' + # The suite runs the gateway with attestation on, so it depends on the + # shared fixture: the simulator, the mock collateral service, and the + # fixture payloads baked into the simulator image. + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'sdk/simulator/**' - '.github/workflows/gateway-proxy-tests.yml' +# Per workflow, not shared with the other two gateway suites. +# +# A group shared across all three does not serialise them, it drops one: a +# concurrency group holds a single *pending* run, so when the third workflow +# queued behind the first two, the one already waiting was cancelled -- +# silently, and reported as a cancelled run rather than a failure. The suites +# no longer need serialising anyway; each brings up its own attestation fixture +# under its own `FIXTURE_NS`, so there is nothing left to collide over. +# +# `cancel-in-progress` is the ordinary meaning here: a new push supersedes the +# run for the commit it replaced. +concurrency: + group: gateway-proxy-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read @@ -32,13 +65,21 @@ jobs: proxy-integration: runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} # Each of the ~25 arms restarts the gateway, and the idle-timeout arms wait - # out a real timeout, so this is minutes rather than seconds. - timeout-minutes: 30 + # out a real timeout. On top of that the suite now builds the shared + # attestation fixture, whose two images compile Rust from a cold docker cache + # on every run -- the cargo cache above does not reach inside a docker build. + timeout-minutes: 45 steps: - uses: actions/checkout@v5 - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 + with: + # The container image is built around a static binary. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools - name: Cache cargo uses: actions/cache@v4 @@ -50,32 +91,51 @@ jobs: key: gateway-proxy-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} restore-keys: gateway-proxy-${{ runner.os }}- - - name: Build the gateway - working-directory: dstack - run: cargo build --release -p dstack-gateway - - name: Record kernel capabilities # The suite adapts to what the kernel offers, so the log needs to say # what it had: a run that skipped kTLS looks the same as one that # covered it otherwise. run: | echo "kernel: $(uname -r)" + # Without this the suite falls back to a dummy link and skips + # test_accel_status entirely -- including the only positive proof that + # splice engaged. The other two workflows already modprobe it. + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" sudo modprobe tls 2>&1 || echo "no TLS ULP available" echo "tls module loaded: $(lsmod | grep -c '^tls ' || true)" grep -B2 -A3 'gcm(aes)' /proc/crypto | grep -E '^(driver|priority)' \ | paste - - | sort -u || true - name: Proxy integration tests - working-directory: dstack/gateway/test-run - env: - GATEWAY_BIN: ${{ github.workspace }}/dstack/target/release/dstack-gateway - run: ./test_proxy.sh + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh + + # No `docker compose logs` here: the suite is driven with `compose run + # --rm`, so its containers are deleted on exit and the script's own trap + # has already run `compose down -v`. Everything worth reading is in the + # bind-mounted work directory, which the suite made host-readable before + # exiting. + - name: Make suite logs readable + if: failure() + working-directory: dstack/gateway/test-run/proxy-e2e + run: | + mkdir -p run + docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true + ls -R run || true - name: Upload logs on failure if: failure() uses: actions/upload-artifact@v4 with: name: gateway-proxy-test-logs - path: /tmp/dstack-gw-proxy-test.*/logs/ + # Only the logs. The work directory also holds each arm's throwaway + # TLS key, and an artifact is the wrong place for key material even + # when it is disposable. + path: dstack/gateway/test-run/proxy-e2e/run/*/logs/ if-no-files-found: ignore retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh down diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 0ac7ac434..25c0e21e2 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -835,12 +835,14 @@ dependencies = [ "dstack-guest-agent-rpc", "dstack-kms-rpc", "dstack-types", + "hex", "http-client", "ra-rpc", "ra-tls", "serde_json", "tdx-attest", "tokio", + "x509-parser 0.16.0", ] [[package]] @@ -5818,6 +5820,7 @@ dependencies = [ "sha2 0.10.9", "sha3", "tdx-attest", + "tempfile", "tokio", "tpm-qvl", "tpm-types", diff --git a/dstack/cert-client/Cargo.toml b/dstack/cert-client/Cargo.toml index 7fe9868cb..e18f43ed4 100644 --- a/dstack/cert-client/Cargo.toml +++ b/dstack/cert-client/Cargo.toml @@ -20,3 +20,8 @@ tdx-attest.workspace = true dstack-guest-agent-rpc.workspace = true http-client = { workspace = true, features = ["prpc"] } tokio.workspace = true + +[dev-dependencies] +hex.workspace = true +x509-parser.workspace = true +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs index e55689b52..93a44909b 100644 --- a/dstack/cert-client/src/lib.rs +++ b/dstack/cert-client/src/lib.rs @@ -9,7 +9,7 @@ use dstack_kms_rpc::{kms_client::KmsClient, SignCertRequest}; use dstack_types::{AppKeys, KeyProvider}; use ra_rpc::client::{RaClient, RaClientConfig}; use ra_tls::{ - attestation::AttestationVerifier, + attestation::{AttestationVerifier, VersionedAttestation}, cert::{generate_ra_cert, CaCert, CertSigningRequestV2}, }; @@ -23,6 +23,38 @@ pub enum CertRequestClient { }, } +/// The app id a locally issued certificate should carry, if any. +/// +/// KMS stamps the app_id it verified into the certificate it returns. The local +/// CA branch has the same value available -- the CSR carries the attestation it +/// was derived from -- but used to drop it, so every certificate issued through +/// a local CA came back without the extension. Consumers that read the +/// extension and have no fallback therefore rejected all of them: +/// `AppIdValidator` in dstack-gateway's cluster sync client and the gateway's +/// own `ensure_from_gateway` both do exactly that, which left clustering +/// working under a KMS key provider and broken under every other one. (The +/// gateway's inbound sync routes happen to fall back to the app-info extension, +/// which is why the failure showed up on one side of the connection only.) +/// +/// Reading the attestation without verifying it is sound *here and only here*: +/// the sole caller is the guest agent signing a CSR it built itself, from its +/// own `certificate_attestation`, against a CA whose key it already holds. It +/// asserts nothing it could not assert anyway. A remote CSR must go through +/// KMS, which verifies the quote first and stamps `boot_info.app_id`. +/// +/// Best effort on purpose: an app whose attestation carries no app-id event +/// decodes to an empty one, and stamping that would make every such peer match +/// every other. Absent stays absent, and the peer rejects it as before. +fn local_ca_app_id(attestation: &VersionedAttestation) -> Option> { + attestation + .clone() + .into_v1() + .decode_app_info(false) + .ok() + .map(|info| info.app_id) + .filter(|app_id| !app_id.is_empty()) +} + impl CertRequestClient { pub async fn sign_csr( &self, @@ -31,8 +63,9 @@ impl CertRequestClient { ) -> Result> { match self { CertRequestClient::Local { ca } => { + let app_id = local_ca_app_id(&csr.attestation); let cert = ca - .sign_csr(csr, None, "app:custom") + .sign_csr(csr, app_id.as_deref(), "app:custom") .context("Failed to sign certificate")?; Ok(vec![cert.pem(), ca.pem_cert.clone()]) } @@ -94,3 +127,169 @@ impl CertRequestClient { } } } + +#[cfg(test)] +mod tests { + use super::*; + use ra_tls::{ + attestation::{Attestation, AttestationQuote, StackEvidence, TdxQuote}, + cert::{CertConfigV2, CertSigningRequestV2}, + rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}, + traits::CertExt, + }; + + /// The guest agent simulator's attestation, the same bytes every gateway + /// suite runs against. Used rather than a hand-built one because the value + /// under test is what `decode_app_info` reads out of a real event log -- + /// a synthetic attestation would prove the plumbing and not the decode. + const SIMULATOR_ATTESTATION: &[u8] = include_bytes!("../../../sdk/simulator/attestation.bin"); + + /// The app id `SIMULATOR_ATTESTATION` decodes to. Written out rather than + /// recomputed, so a change in either the fixture or the decode shows up + /// here as a failure instead of being silently agreed with. + const SIMULATOR_APP_ID: &str = "5bb4ff9a3837357f19dc176407a5709c62eb6c56"; + + fn simulator_attestation() -> VersionedAttestation { + VersionedAttestation::from_bytes(SIMULATOR_ATTESTATION).expect("decode fixture") + } + + /// The simulator's attestation with the app-id event's payload emptied. + /// + /// This is the case the `.filter()` in `local_ca_app_id` exists for and the + /// only one that reaches it: `find_event_payload` returns an empty vec for a + /// missing or empty payload, so `decode_app_info` SUCCEEDS and hands back an + /// AppInfo whose app_id is empty. The undecodable attestation below does not + /// reach the filter at all -- it fails earlier, at `.ok()` -- so it cannot + /// stand in for this. Only the digest is measured, so clearing the payload + /// leaves the RTMR replay intact. + fn attestation_with_empty_app_id() -> VersionedAttestation { + let mut attestation = simulator_attestation().into_v1(); + let StackEvidence::Dstack { + ref mut runtime_events, + .. + } = attestation.stack + else { + panic!("the simulator fixture is expected to carry dstack stack evidence"); + }; + let mut found = false; + for event in runtime_events.iter_mut() { + if event.event == "app-id" { + event.payload = Vec::new(); + found = true; + } + } + assert!( + found, + "fixture must carry an app-id event for this to mean anything" + ); + VersionedAttestation::V1 { attestation } + } + + /// An attestation carrying nothing decodable: the case where no app id can + /// be read at all. + fn undecodable_attestation() -> VersionedAttestation { + Attestation { + quote: AttestationQuote::DstackTdx(TdxQuote { + quote: vec![], + event_log: vec![], + }), + runtime_events: vec![], + report_data: [0u8; 64], + config: "".into(), + report: (), + } + .into_versioned() + } + + fn csr_with(attestation: VersionedAttestation, pubkey: Vec) -> CertSigningRequestV2 { + CertSigningRequestV2 { + confirm: "please sign cert:".to_string(), + pubkey, + config: CertConfigV2 { + org_name: None, + subject: "local-ca-test".to_string(), + subject_alt_names: vec![], + usage_server_auth: true, + usage_client_auth: true, + ext_quote: false, + // Deliberately off. `sign_csr` derives app *info* separately + // when this is set, and hard fails when it cannot -- which + // would mask the app *id* path this is about. The gateway does + // set it, which is why the break showed on the client side + // only; see `local_ca_app_id`. + ext_app_info: false, + not_before: None, + not_after: None, + }, + attestation, + } + } + + fn local_client() -> CertRequestClient { + let key = KeyPair::generate().expect("ca key"); + let mut params = CertificateParams::new(vec![]).expect("ca params"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let cert = params.self_signed(&key).expect("ca cert"); + let ca = CaCert::new(cert.pem(), key.serialize_pem()).expect("ca"); + CertRequestClient::Local { ca: Box::new(ca) } + } + + /// Sign through the real `CertRequestClient::Local` branch and read the + /// app-id extension back off the leaf, the way a peer does. + async fn issued_app_id(attestation: VersionedAttestation) -> Option> { + let leaf_key = KeyPair::generate().expect("leaf key"); + let csr = csr_with(attestation, leaf_key.public_key_der()); + let chain = local_client() + .sign_csr(&csr, &[]) + .await + .expect("local signing"); + let (_, leaf) = x509_parser::pem::parse_x509_pem(chain[0].as_bytes()).expect("leaf pem"); + let (_, leaf) = x509_parser::parse_x509_certificate(&leaf.contents).expect("leaf der"); + leaf.get_app_id().expect("read app id") + } + + /// The contract dstack-gateway's cluster mTLS rests on. + /// + /// `AppIdValidator` (the sync client) and `ensure_from_gateway` (the RPC + /// handler) read this extension with no fallback, so a locally issued + /// certificate without it is rejected by every peer. Reverting the stamp to + /// the `None` it used to pass turns this red. + #[tokio::test] + async fn a_locally_issued_certificate_carries_the_attested_app_id() { + let app_id = issued_app_id(simulator_attestation()).await; + assert_eq!( + app_id.as_deref().map(hex::encode).as_deref(), + Some(SIMULATOR_APP_ID), + "a peer that pins app_id must be able to read it off a local CA's certificate" + ); + } + + /// And what must NOT happen: an attestation yielding nothing leaves the + /// extension absent rather than stamping an empty value. + /// + /// An empty app id in the extension is not "unknown", it is a value, and + /// both consumers compare by equality -- so every app without an app-id + /// event would have matched every other one. Dropping the `.filter()` in + /// `local_ca_app_id` turns this red. + #[tokio::test] + async fn an_attestation_with_no_app_id_leaves_the_extension_absent() { + assert_eq!( + issued_app_id(attestation_with_empty_app_id()).await, + None, + "an empty app id must be absent, not stamped" + ); + assert_eq!(issued_app_id(undecodable_attestation()).await, None); + } + + /// The same rule at the unit it is decided in, so a failure says which of + /// the two halves moved. + #[test] + fn local_ca_app_id_is_absent_rather_than_empty() { + assert_eq!(local_ca_app_id(&attestation_with_empty_app_id()), None); + assert_eq!(local_ca_app_id(&undecodable_attestation()), None); + assert_eq!( + local_ca_app_id(&simulator_attestation()).map(hex::encode), + Some(SIMULATOR_APP_ID.to_string()) + ); + } +} diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs index 56034b1d1..efa1652bc 100644 --- a/dstack/certbot/src/acme_client.rs +++ b/dstack/certbot/src/acme_client.rs @@ -261,6 +261,24 @@ pub fn advisory_dns_wait(configured: Duration, renew_timeout: Duration) -> Durat configured.min(capped) } +/// How long the DNS poll may sleep before its next lookup. +/// +/// The backoff, cut to whatever is left of the budget. Split out from the loop +/// because the loop cannot be driven from a test -- it needs a resolver and a +/// live challenge -- while the arithmetic is the whole of the rule and is what +/// went wrong: sleeping the full backoff and checking the budget afterwards +/// overshoots by up to a whole step (32s at the top of the ramp), which for a +/// short `renew_timeout` hands the deadline to the timeout wrapping the order. +/// The graceful "proceed anyway" exit is then missed and the renewal ends as a +/// bare timeout, naming nothing. +/// +/// A zero budget yields a zero sleep, and the caller's expiry check fires on the +/// same pass, so `max_dns_wait = 0` means the check is skipped outright rather +/// than run once. +fn dns_poll_sleep(budget: Duration, elapsed: Duration, backoff: Duration) -> Duration { + budget.saturating_sub(elapsed).min(backoff) +} + /// A AcmeClient instance. pub struct AcmeClient { account: Account, @@ -929,10 +947,9 @@ impl AcmeClient { // `renew_timeout` is enough to hand the deadline to the timeout // wrapping the order, so the graceful exit below is missed and the // renewal ends as a timeout with nothing named. - let elapsed = start_time.elapsed(); - let remaining = self.max_dns_wait.saturating_sub(elapsed); - if !remaining.is_zero() { - sleep(delay.min(remaining)).await; + let nap = dns_poll_sleep(self.max_dns_wait, start_time.elapsed(), delay); + if !nap.is_zero() { + sleep(nap).await; } let elapsed = start_time.elapsed(); @@ -1768,26 +1785,172 @@ mod caa_guard_tests { } } +/// The DNS wait budget: what `max_dns_wait` is clamped to, and what the poll +/// loop does with the result. +/// +/// Both halves are pure and both were uncovered. `advisory_dns_wait` exists +/// only for the clamp, and nothing asserted it; `dns_poll_sleep` is the rule the +/// loop's own comment describes and could not be reached without a resolver and +/// a live challenge. #[cfg(test)] mod dns_wait_tests { - use super::advisory_dns_wait; + use super::{advisory_dns_wait, dns_poll_sleep, DNS_WAIT_SHARE_OF_RENEW_TIMEOUT}; use std::time::Duration; + const fn secs(n: u64) -> Duration { + Duration::from_secs(n) + } + + /// A wait already inside the budget is left where the operator put it: the + /// clamp is a ceiling, not an override. + #[test] + fn a_wait_that_already_fits_is_unchanged() { + assert_eq!(advisory_dns_wait(secs(30), secs(600)), secs(30)); + assert_eq!(advisory_dns_wait(secs(5), secs(300)), secs(5)); + } + /// The CLI's own defaults are the failing case: a 300s wait inside a 120s /// renewal budget never reaches its "proceed anyway" exit, so an unanswered /// check ends as a timeout rather than as the warning naming the record. #[test] fn the_wait_ends_before_the_renewal_budget_does() { - let wait = advisory_dns_wait(Duration::from_secs(300), Duration::from_secs(120)); - assert!(wait < Duration::from_secs(120), "{wait:?}"); + let renew = secs(120); + let wait = advisory_dns_wait(secs(300), renew); + assert_eq!(wait, secs(60)); + assert!(wait < renew, "{wait:?}"); } - /// A wait already inside the budget is left where the operator put it. + /// The boundary belongs to the configured value: exactly half the renewal + /// budget still fits. #[test] - fn a_wait_that_already_fits_is_unchanged() { + fn a_wait_of_exactly_half_the_renewal_budget_fits() { + assert_eq!(advisory_dns_wait(secs(150), secs(300)), secs(150)); + } + + /// Over the ceiling, the ceiling wins. + #[test] + fn a_wait_that_does_not_fit_is_cut_to_the_ceiling() { + assert_eq!(advisory_dns_wait(secs(300), secs(60)), secs(30)); + assert_eq!(advisory_dns_wait(secs(3600), secs(300)), secs(150)); + } + + /// The case the clamp was written for: the stock defaults on both sides are + /// 300s, so without it the wait and the timeout wrapping the order expire + /// together and the outer one always wins -- the renewal dies as a bare + /// timeout and the warning naming the missing record is never logged. + /// + /// What matters is not the number but that the wait ends strictly first. + #[test] + fn the_stock_defaults_leave_room_for_the_graceful_exit() { + let renew = secs(300); + let wait = advisory_dns_wait(secs(300), renew); + assert_eq!(wait, secs(150)); + assert!( + wait < renew, + "the DNS wait must end before the timeout wrapping the order" + ); + } + + /// Zero configured means the check is off, and the clamp must not quietly + /// turn it back on. + /// + /// Reachable from the certbot CLI, whose `max_dns_wait` is a plain + /// `#[serde(default)]` field with no lower bound -- unlike the gateway's + /// `CreateDnsCredential`, which refuses zero with "max_dns_wait must be + /// greater than zero". So the clamp is the only thing standing between a + /// zero in a config file and whatever the poll loop would do with it, and + /// what it must do is pass the zero through: the loop's own reading of a + /// spent budget is what turns the check off, and a clamp that rounded zero + /// up would re-enable a check the operator switched off. + #[test] + fn zero_stays_zero() { + assert_eq!(advisory_dns_wait(secs(0), secs(300)), secs(0)); + } + + /// A renewal budget of zero leaves nothing for the wait, and must not + /// underflow or panic on the division. + #[test] + fn a_zero_renewal_budget_leaves_no_wait() { + assert_eq!(advisory_dns_wait(secs(300), secs(0)), secs(0)); + assert_eq!(advisory_dns_wait(secs(0), secs(0)), secs(0)); + } + + /// The two invariants, over the whole grid rather than at the points above: + /// never more than what was configured, and never more than the share of the + /// renewal budget the wait is allowed. + #[test] + fn the_result_never_exceeds_either_bound() { + for configured in [0, 1, 5, 59, 60, 61, 150, 299, 300, 3600] { + for renew in [0, 1, 2, 60, 119, 120, 300, 301] { + let (configured, renew) = (secs(configured), secs(renew)); + let got = advisory_dns_wait(configured, renew); + assert!(got <= configured, "{got:?} > configured {configured:?}"); + assert!( + got <= renew / DNS_WAIT_SHARE_OF_RENEW_TIMEOUT, + "{got:?} exceeds its share of {renew:?}" + ); + } + } + } + + /// With budget to spare, the poll sleeps the full backoff step. + #[test] + fn a_poll_with_budget_to_spare_sleeps_the_whole_backoff() { + assert_eq!( + dns_poll_sleep(secs(150), secs(10), Duration::from_millis(250)), + Duration::from_millis(250) + ); + assert_eq!(dns_poll_sleep(secs(150), secs(10), secs(32)), secs(32)); + } + + /// Near the end of the budget the sleep is cut to what is left. + /// + /// The regression this guards: sleeping the full step and checking after + /// overshoots by up to 32s, which is enough to hand the deadline to the + /// timeout wrapping the order. + #[test] + fn a_poll_near_the_deadline_sleeps_only_what_is_left() { + assert_eq!(dns_poll_sleep(secs(60), secs(58), secs(32)), secs(2)); assert_eq!( - advisory_dns_wait(Duration::from_secs(30), Duration::from_secs(600)), - Duration::from_secs(30) + dns_poll_sleep(secs(5), secs(4), secs(32)), + secs(1), + "the e2e suite's 5s budget must not be overrun by a 32s backoff step" ); } + + /// A spent budget sleeps not at all, so the caller reaches its expiry check + /// on the same pass. + #[test] + fn a_spent_budget_sleeps_not_at_all() { + assert_eq!(dns_poll_sleep(secs(60), secs(60), secs(32)), secs(0)); + assert_eq!(dns_poll_sleep(secs(60), secs(90), secs(32)), secs(0)); + } + + /// `max_dns_wait = 0` skips the check rather than running it once: the first + /// pass sleeps nothing and the caller's `elapsed >= budget` is already true, + /// so no lookup is ever made. + #[test] + fn a_zero_budget_skips_the_check_entirely() { + let budget = secs(0); + assert_eq!(dns_poll_sleep(budget, secs(0), secs(32)), secs(0)); + assert!( + secs(0) >= budget, + "the caller's expiry check fires on the first pass" + ); + } + + /// The sleep never runs past the budget, for any point inside it. + #[test] + fn a_poll_never_sleeps_past_the_budget() { + for budget in [0u64, 1, 5, 60, 150] { + for elapsed in 0..=budget + 2 { + for backoff in [1u64, 2, 4, 8, 16, 32] { + let (b, e) = (secs(budget), secs(elapsed)); + let nap = dns_poll_sleep(b, e, secs(backoff)); + assert!(e + nap <= b.max(e), "{e:?} + {nap:?} overruns {b:?}"); + assert!(nap <= secs(backoff)); + } + } + } + } } diff --git a/dstack/gateway/Cargo.toml b/dstack/gateway/Cargo.toml index 0058ec068..e682bb690 100644 --- a/dstack/gateway/Cargo.toml +++ b/dstack/gateway/Cargo.toml @@ -74,6 +74,9 @@ socket2.workspace = true [dev-dependencies] insta.workspace = true +# The test-only CA/leaf helpers. A dev-dependency on the same crate the normal +# dependency points at, so the feature is enabled for tests and not for the binary. +ra-tls = { workspace = true, features = ["test-pki"] } rmpv.workspace = true tempfile.workspace = true wavekv-v1 = { package = "wavekv", version = "=1.0.0" } diff --git a/dstack/gateway/src/config.rs b/dstack/gateway/src/config.rs index fdab616ac..fa587bc28 100644 --- a/dstack/gateway/src/config.rs +++ b/dstack/gateway/src/config.rs @@ -50,8 +50,8 @@ impl WgConfig { /// `client_ip_range`. Nothing in this node's config describes the other /// nodes' pools, and the deployments do not even agree on a shape that /// could be inferred: `dstack-app/deploy-to-vmm.sh` puts every pool inside - /// one /16 that each interface covers, while `test-run/cluster.sh` and the - /// e2e configs give each node a /24 that no other node's interface covers. + /// one /16 that each interface covers, while the `test-run` suites give + /// each node a /24 that no other node's interface covers. /// Judging a replicated address by local topology refuses legitimate peers /// under the second shape, so this is limited to what a node can assert on /// its own: an ordinary unicast address that is not one of *this* gateway's. diff --git a/dstack/gateway/src/kv/import.rs b/dstack/gateway/src/kv/import.rs index 1b49d3023..593715379 100644 --- a/dstack/gateway/src/kv/import.rs +++ b/dstack/gateway/src/kv/import.rs @@ -419,8 +419,8 @@ mod tests { let shapes = [ // deploy-to-vmm.sh: /18 pools inside a shared /16 interface. ("10.8.0.1/16", "10.8.0.0/18", "10.8.0.5", "10.8.64.5"), - // test-run/cluster.sh and e2e/configs: a /24 per node, and no - // node's interface covers another's. + // The test-run suites: a /24 per node, and no node's interface + // covers another's. ("10.0.41.1/24", "10.0.41.0/24", "10.0.41.5", "10.0.42.5"), ]; for (ip, pool, mine, peers) in shapes { diff --git a/dstack/gateway/src/main_service/tests.rs b/dstack/gateway/src/main_service/tests.rs index ca5524499..8d60abcc7 100644 --- a/dstack/gateway/src/main_service/tests.rs +++ b/dstack/gateway/src/main_service/tests.rs @@ -2133,8 +2133,8 @@ async fn a_poisoned_peer_record_costs_only_its_own_instance() { #[tokio::test] async fn a_cvm_registered_on_another_node_becomes_a_wg_peer_here() { let state = create_test_state().await; - // What a peer node allocated out of its own slice. `test-run/cluster.sh` - // and the e2e configs give each node a /24 of its own, so a peer's address + // What a peer node allocated out of its own slice. The `test-run` suites + // give each node a /24 of its own, so a peer's address // is outside this node's pool *and* outside its interface network — yet // every CVM is handed every gateway as a WireGuard server, so this node // still has to carry it. diff --git a/dstack/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs index b1a00bf51..7c7860884 100644 --- a/dstack/gateway/src/web_routes/wavekv_sync.rs +++ b/dstack/gateway/src/web_routes/wavekv_sync.rs @@ -72,8 +72,14 @@ async fn read_compressed_body(data: Data<'_>) -> Result, Status> { } /// Read a sync envelope from a bounded request body. -async fn read_envelope(data: Data<'_>) -> Result { - let decompressed = gunzip(&read_compressed_body(data).await?)?; +/// Decode a gzipped sync envelope. +/// +/// Split from the `Data` read so the request-handling half of these endpoints +/// can be driven from a test. Rocket's local client presents no certificate, so +/// a test that went through the real route would be stopped at the peer check +/// before reaching any of this. +fn decode_envelope(compressed: &[u8]) -> Result { + let decompressed = gunzip(compressed)?; // `SyncEnvelope::decode` enforces the schema version and rejects trailing bytes; // it is deliberately not the generic `decode` used for KV values. SyncEnvelope::decode(&decompressed).map_err(|e| { @@ -102,23 +108,26 @@ fn verify_gateway_peer(state: &Proxy, cert: Option>) -> Result<( /// Split out from `verify_gateway_peer` because that function's other half — the /// attestation bypass and Rocket's certificate guard — cannot be exercised from a test, /// which left this decision, the actual authorization rule, uncovered. +/// +/// The app-id extension is the only identity accepted. There used to be a fallback to +/// the app-info extension here, and it existed because certificates issued through a +/// local CA carried app-info but not app-id — so without it, clustering under any +/// non-KMS key provider failed. That is fixed at the source now (`local_ca_app_id` in +/// cert-client stamps the app id the way KMS does), which matters because the fallback +/// only ever covered *this* side of the connection: `AppIdValidator` in the sync client +/// and `ensure_from_gateway` in the RPC handler both read app-id with no fallback, so a +/// certificate the fallback rescued here was still rejected in both other places. One +/// rule across all three is worth more than a second path that was untested, disagreed +/// with its neighbours, and read an issuer claim the cluster does not otherwise +/// authorize on. fn authorize_peer(cert: &impl CertExt, my_app_id: Option<&[u8]>) -> Result<(), Status> { - let remote_app_id = match cert.get_app_id().map_err(|e| { + let remote_app_id = cert.get_app_id().map_err(|e| { warn!("WaveKV sync: failed to extract app_id from certificate: {e}"); Status::Unauthorized - })? { - Some(app_id) => Some(app_id), - None => cert - .get_app_info() - .map_err(|e| { - warn!("WaveKV sync: failed to extract app_info from certificate: {e}"); - Status::Unauthorized - })? - .map(|info| info.app_id), - }; + })?; let Some(remote_app_id) = remote_app_id else { - warn!("WaveKV sync: certificate does not contain app identity"); + warn!("WaveKV sync: certificate does not contain an app_id extension"); return Err(Status::Unauthorized); }; @@ -139,12 +148,21 @@ pub async fn sync_store( data: Data<'_>, ) -> Result<(ContentType, Vec), Status> { verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) +} +/// Everything `sync_store` does once the caller is known to be a peer. +pub(crate) fn handle_sync( + state: &Proxy, + store: &str, + body: &[u8], +) -> Result<(ContentType, Vec), Status> { let Some(ref wavekv_sync) = state.wavekv_sync else { return Err(Status::ServiceUnavailable); }; - let env = read_envelope(data).await?; + let env = decode_envelope(body)?; if env.sender_id == 0 { warn!("rejected sync from invalid node_id 0"); return Err(Status::BadRequest); @@ -199,12 +217,17 @@ pub async fn push_store( data: Data<'_>, ) -> Result { verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) +} +/// Everything `push_store` does once the caller is known to be a peer. +pub(crate) fn handle_push(state: &Proxy, store: &str, body: &[u8]) -> Result { let Some(ref wavekv_sync) = state.wavekv_sync else { return Err(Status::ServiceUnavailable); }; - let env = read_envelope(data).await?; + let env = decode_envelope(body)?; if env.sender_id == 0 { warn!("rejected push from invalid node_id 0"); return Err(Status::BadRequest); @@ -233,6 +256,9 @@ mod tests { const ME: u32 = 1; const PEER: u32 = 2; + /// Both sides of the local mTLS test are this gateway, so one id serves for + /// the certificate it presents and the identity it expects of a peer. + const TEST_APP_ID: &[u8] = b"test-app-id-0000-0001"; fn peer_uuid() -> Vec { b"the-real-peer-2".to_vec() @@ -241,58 +267,61 @@ mod tests { /// A self-signed CA plus a leaf it signs. `HttpSyncNetwork::new` loads all three /// from disk to build its rustls client config, and the root store only accepts a /// trust anchor with `CA:TRUE` — so a lone self-signed leaf is not enough. + /// + /// The leaf carries an app_id because no test here turns the peer check off, so a + /// certificate without one is refused before any of them reach what they are about. + /// It is also what a real peer's certificate carries. fn write_tls_material(dir: &std::path::Path) -> TlsConfig { - use ra_tls::rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; - - let ca_key = KeyPair::generate().expect("ca key"); - let mut ca_params = CertificateParams::new(vec![]).expect("ca params"); - ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); - let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); - - let leaf_key = KeyPair::generate().expect("leaf key"); - let leaf_params = - CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); - let leaf_cert = leaf_params - .signed_by(&leaf_key, &ca_cert, &ca_key) - .expect("leaf cert"); - - let cert_path = dir.join("node.crt"); - let key_path = dir.join("node.key"); - let ca_path = dir.join("ca.crt"); - std::fs::write(&cert_path, leaf_cert.pem()).expect("write cert"); - std::fs::write(&key_path, leaf_key.serialize_pem()).expect("write key"); - std::fs::write(&ca_path, ca_cert.pem()).expect("write ca"); + let pki = ra_tls::test_pki::write_mtls_pki( + dir, + ra_tls::test_pki::TestCert::localhost().app_id(TEST_APP_ID), + ) + .expect("write test PKI"); TlsConfig { - certs: cert_path.to_string_lossy().into_owned(), - key: key_path.to_string_lossy().into_owned(), + certs: pki.cert_path.to_string_lossy().into_owned(), + key: pki.key_path.to_string_lossy().into_owned(), mutual: MutualConfig { - ca_certs: ca_path.to_string_lossy().into_owned(), + ca_certs: pki.ca_cert_path.to_string_lossy().into_owned(), }, } } - /// A gateway serving the real sync routes over Rocket's local client. + /// A gateway whose request-handling half these tests drive directly. /// - /// `insecure_skip_attestation` is on, which makes `verify_gateway_peer` return - /// immediately: these tests are about everything below it — route dispatch, the gzip - /// framing, the store split, the uuid check. `enforcing_gateway` covers the gate - /// itself, which this fixture cannot, because Rocket's local client speaks no TLS - /// and so can never present a certificate. - async fn serving_gateway(sync_enabled: bool) -> (Client, Proxy, TempDir) { - serving_gateway_with(sync_enabled, true).await + /// They are about everything below the peer check — the gzip framing, the store + /// split, the uuid check, the removed-sender refusal — and they call `handle_sync` + /// and `handle_push` rather than going through the routes, because the peer check runs + /// on every request these tests make and Rocket's local client speaks no TLS, so a + /// request through the route can never get past it. `enforcing_gateway` covers the check itself. + async fn serving_gateway(sync_enabled: bool) -> (Proxy, TempDir) { + let (_client, proxy, tmp) = serving_gateway_with(sync_enabled).await; + (proxy, tmp) + } + + /// The same gateway, reached through the real routes, where every request is refused + /// for want of a certificate. That refusal is the assertion. + async fn enforcing_gateway() -> (Client, Proxy, TempDir) { + serving_gateway_with(true).await } - /// The same gateway with the attestation bypass switched off, so the peer check runs - /// for real. - async fn enforcing_gateway() -> (Client, Proxy, TempDir) { - serving_gateway_with(true, false).await + /// Drive the sync endpoint's body the way its route would. + fn post_sync(proxy: &Proxy, store: &str, body: Vec) -> (Status, Vec) { + match handle_sync(proxy, store, &body) { + Ok((_content_type, bytes)) => (Status::Ok, bytes), + Err(status) => (status, Vec::new()), + } + } + + /// Drive the push endpoint's body the way its route would. + fn post_push(proxy: &Proxy, store: &str, body: Vec) -> Status { + match handle_push(proxy, store, &body) { + Ok(status) => status, + Err(status) => status, + } } - async fn serving_gateway_with( - sync_enabled: bool, - skip_attestation: bool, - ) -> (Client, Proxy, TempDir) { + async fn serving_gateway_with(sync_enabled: bool) -> (Client, Proxy, TempDir) { // `main` installs this once at startup; the sync client builds a rustls config, // so a test that skips it panics inside rustls rather than failing an assertion. let _ = rustls::crypto::ring::default_provider().install_default(); @@ -310,12 +339,11 @@ mod tests { .join("wg.conf") .to_string_lossy() .into_owned(); - config.debug.insecure_skip_attestation = skip_attestation; let tls_config = write_tls_material(temp_dir.path()); let proxy = Proxy::new(ProxyOptions { config, - my_app_id: None, + my_app_id: Some(TEST_APP_ID.to_vec()), tls_config, }) .await @@ -330,10 +358,9 @@ mod tests { /// The sync routes are the cluster's write surface: anything that reaches them can /// insert entries that replicate to every gateway. `verify_gateway_peer` is the only - /// thing standing in front of them, and with `insecure_skip_attestation` set — which - /// every other test here sets — its first statement returns `Ok(())`, so the gate - /// itself was never executed by any test. Replacing the whole function body with - /// `Ok(())` did not turn the suite red. + /// thing standing in front of them, and no test had ever executed it: every test + /// here reached the body below by turning the check off. Replacing the whole + /// function body with `Ok(())` did not turn the suite red. /// /// Rocket's local client speaks no TLS and so presents no certificate, which is /// exactly the case that must be refused. @@ -357,33 +384,17 @@ mod tests { /// `CertRequest` adds unconditionally, and the check under test never looks at a /// quote — it reads two extensions and compares bytes. fn cert_with_app_id(app_id: &[u8]) -> Vec { - use ra_tls::cert::CertRequest; - use ra_tls::rcgen::KeyPair; - - let key = KeyPair::generate().expect("key"); - let cert = CertRequest::builder() - .key(&key) - .subject("peer.test") + ra_tls::test_pki::TestCert::new("peer.test") .app_id(app_id) - .build() - .self_signed() - .expect("self-signed cert"); - cert.der().to_vec() + .self_signed_der() + .expect("self-signed cert") } /// A certificate with no app identity at all. fn cert_without_app_id() -> Vec { - use ra_tls::cert::CertRequest; - use ra_tls::rcgen::KeyPair; - - let key = KeyPair::generate().expect("key"); - let cert = CertRequest::builder() - .key(&key) - .subject("peer.test") - .build() - .self_signed() - .expect("self-signed cert"); - cert.der().to_vec() + ra_tls::test_pki::TestCert::new("peer.test") + .self_signed_der() + .expect("self-signed cert") } fn authorize(der: &[u8], my_app_id: Option<&[u8]>) -> Result<(), Status> { @@ -395,7 +406,7 @@ mod tests { /// The rule the sync routes are defended by: same app id or nothing. /// /// Every case below was previously unreachable, because the only tests that touched - /// this code set `insecure_skip_attestation` and returned before it. Inverting the + /// this code returned before it without checking anything. Inverting the /// comparison to `==` left the suite green. #[test] fn a_peer_is_authorized_only_when_its_app_id_matches_ours() { @@ -410,6 +421,32 @@ mod tests { ); } + /// App info is not app id. + /// + /// This check used to fall back to the app-info extension when app-id was absent, + /// which made it the only one of the cluster's three identity checks that would + /// accept such a certificate -- `AppIdValidator` in the sync client and + /// `ensure_from_gateway` in the RPC handler both refuse it. The fallback existed to + /// paper over locally issued certificates that carried no app id; that is fixed + /// where it is caused now, in cert-client. + /// + /// Restoring the fallback turns this red. Nothing did before: the branch had no + /// test at all, and deleting it left all 308 gateway tests green. + #[test] + fn app_info_alone_does_not_authorize_a_peer() { + let ours = b"app-id-of-this-cluster".to_vec(); + let cert = ra_tls::test_pki::TestCert::new("peer.test") + .app_info(&ours) + .self_signed_der() + .expect("self-signed cert"); + + assert_eq!( + authorize(&cert, Some(&ours)), + Err(Status::Unauthorized), + "the app-id extension is the only identity the sync routes accept" + ); + } + /// A certificate that proves nothing about which app presented it is refused, rather /// than falling through to a comparison against `None`. #[test] @@ -488,16 +525,16 @@ mod tests { #[tokio::test] async fn a_stamped_push_is_accepted_and_lands_in_the_store() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/push/persistent") - .body(body(&push_envelope(peer_uuid(), "node/9"))) - .dispatch() - .await; + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(peer_uuid(), "node/9")), + ); - assert_eq!(response.status(), Status::Ok); + assert_eq!(status, Status::Ok); assert!( proxy.kv_store().persistent().read().get("node/9").is_some(), "a well-formed push must reach the store" @@ -509,16 +546,16 @@ mod tests { /// `check_uuid` — which only the manager runs, not `merge_push` — rejected it. #[tokio::test] async fn an_unstamped_push_is_refused_at_the_route() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/push/persistent") - .body(body(&push_envelope(Vec::new(), "node/9"))) - .dispatch() - .await; + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(Vec::new(), "node/9")), + ); - assert_eq!(response.status(), Status::InternalServerError); + assert_eq!(status, Status::InternalServerError); assert!( proxy.kv_store().persistent().read().get("node/9").is_none(), "a push that fails the identity check must not write anything" @@ -527,7 +564,7 @@ mod tests { #[tokio::test] async fn a_sync_round_trip_returns_a_decodable_envelope() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); proxy .kv_store() @@ -537,14 +574,9 @@ mod tests { .expect("seed"); let request = SyncEnvelope::new(PEER, peer_uuid()); - let response = client - .post("/wavekv/sync/persistent") - .body(body(&request)) - .dispatch() - .await; - - assert_eq!(response.status(), Status::Ok); - let bytes = response.into_bytes().await.expect("body"); + let (status, bytes) = post_sync(&proxy, "persistent", body(&request)); + + assert_eq!(status, Status::Ok); let decoded = SyncEnvelope::decode(&gunzip(&bytes).expect("gunzip")).expect("decode"); assert_eq!(decoded.sender_id, ME); assert!( @@ -560,7 +592,7 @@ mod tests { async fn sync_and_push_cross_a_real_mutually_authenticated_tls_connection() { use rocket::{mtls::MtlsConfig, tls::TlsConfig as RocketTlsConfig}; - let (_local, proxy, tmp) = serving_gateway(true).await; + let (proxy, tmp) = serving_gateway(true).await; register_peer(&proxy); let tls = write_tls_material(tmp.path()); @@ -651,9 +683,42 @@ mod tests { server.await.expect("server task"); } + /// The production routes with the peer check removed. + /// + /// Mounted by one test, and only because the body cap it asserts lives in + /// the route half: `handle_sync` never sees a `Data`, so a test calling it + /// directly cannot reach the limit, and Rocket's local client cannot present + /// the certificate the real route now requires. Two duplicated lines are + /// cheaper than losing coverage of a bound that exists to stop a peer -- or a + /// stranger who got that far -- from making the gateway read without end. + #[post("/wavekv/sync/", data = "")] + async fn ungated_sync( + state: &State, + store: &str, + data: Data<'_>, + ) -> Result<(ContentType, Vec), Status> { + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) + } + + #[post("/wavekv/push/", data = "")] + async fn ungated_push( + state: &State, + store: &str, + data: Data<'_>, + ) -> Result { + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) + } + #[tokio::test] async fn an_oversized_compressed_request_is_rejected_explicitly() { - let (client, _proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; + let rocket = rocket::build() + .manage(proxy) + .mount("/", rocket::routes![ungated_sync, ungated_push]); + let client = Client::tracked(rocket).await.expect("rocket client"); + for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { let response = client .post(path) @@ -667,16 +732,12 @@ mod tests { /// Unknown stores are rejected rather than being routed to either replicated store. #[tokio::test] async fn an_unknown_store_is_rejected() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/sync/bogus") - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; + let (status, _) = post_sync(&proxy, "bogus", body(&SyncEnvelope::new(PEER, peer_uuid()))); - assert_eq!(response.status(), Status::NotFound); + assert_eq!(status, Status::NotFound); } /// The lockout at the door. The app-identity check proves the sender is @@ -686,53 +747,65 @@ mod tests { /// SetNodeUrl re-admission path -- opens the door again. #[tokio::test] async fn a_removed_nodes_envelopes_are_refused_at_the_door() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); proxy.kv_store().mark_peer_removed(PEER).expect("mark"); - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client - .post(path) - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::Forbidden, - "{path} must refuse a removed sender" - ); - } + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::Forbidden, + "sync must refuse a removed sender" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::Forbidden, + "push must refuse a removed sender" + ); proxy.kv_store().clear_peer_removed(PEER).expect("clear"); - let response = client - .post("/wavekv/sync/persistent") - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::Ok, - "re-admission opens the door again" + let (readmitted, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), ); + assert_eq!(readmitted, Status::Ok, "re-admission opens the door again"); } /// A node with synchronization disabled reports that the service is unavailable. #[tokio::test] async fn a_sync_disabled_node_answers_503() { - let (client, _proxy, _tmp) = serving_gateway(false).await; + let (proxy, _tmp) = serving_gateway(false).await; - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client - .post(path) - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::ServiceUnavailable, - "{path} must not look like a missing v2 route" - ); - } + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::ServiceUnavailable, + "sync must not look like a missing v2 route" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::ServiceUnavailable, + "push must not look like a missing v2 route" + ); } /// gzip expands by three orders of magnitude on attacker-chosen input, so the @@ -741,7 +814,7 @@ mod tests { /// has to hold against a peer running a buggy build, not just against a stranger. #[tokio::test] async fn a_compression_bomb_is_refused_before_it_is_decompressed() { - let (client, _proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; // ~130 MiB of zeroes compresses to well under the request cap. let bomb = gzip(&vec![0u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); @@ -751,14 +824,18 @@ mod tests { bomb.len() ); - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client.post(path).body(bomb.clone()).dispatch().await; - assert_eq!( - response.status(), - Status::BadRequest, - "{path} must refuse an over-sized expansion" - ); - } + let (sync_status, _) = post_sync(&proxy, "persistent", bomb.clone()); + assert_eq!( + sync_status, + Status::BadRequest, + "sync must refuse an over-sized expansion" + ); + let push_status = post_push(&proxy, "persistent", bomb); + assert_eq!( + push_status, + Status::BadRequest, + "push must refuse an over-sized expansion" + ); } /// The limits must leave room for the largest legitimate message. @@ -805,17 +882,13 @@ mod tests { #[tokio::test] async fn a_push_from_node_id_zero_is_refused() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); let mut env = push_envelope(peer_uuid(), "node/9"); env.sender_id = 0; - let response = client - .post("/wavekv/push/persistent") - .body(body(&env)) - .dispatch() - .await; + let status = post_push(&proxy, "persistent", body(&env)); - assert_eq!(response.status(), Status::BadRequest); + assert_eq!(status, Status::BadRequest); } } diff --git a/dstack/gateway/test-run/.env.example b/dstack/gateway/test-run/.env.example deleted file mode 100644 index ff6571750..000000000 --- a/dstack/gateway/test-run/.env.example +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Cloudflare API token with DNS edit permissions -# Required scopes: Zone.DNS (Edit), Zone.Zone (Read) -CF_API_TOKEN=your_cloudflare_api_token_here - -# Cloudflare Zone ID for your domain -CF_ZONE_ID=your_zone_id_here - -# Test domain (must be a wildcard domain managed by Cloudflare) -# Example: *.test.example.com -TEST_DOMAIN=*.test.example.com diff --git a/dstack/gateway/test-run/.gitignore b/dstack/gateway/test-run/.gitignore index b1c90f813..ba4042005 100644 --- a/dstack/gateway/test-run/.gitignore +++ b/dstack/gateway/test-run/.gitignore @@ -1,4 +1,12 @@ /run/ .env /e2e/dstack-gateway +/cluster/dstack-gateway +/cluster/run/ __pycache__/ +/proxy-e2e/dstack-gateway +/proxy-e2e/test_proxy.sh +/proxy-e2e/proxy/ +/proxy-e2e/run/ +*.lock +.suite.lock diff --git a/dstack/gateway/test-run/Dockerfile.gateway b/dstack/gateway/test-run/Dockerfile.gateway new file mode 100644 index 000000000..66cc90769 --- /dev/null +++ b/dstack/gateway/test-run/Dockerfile.gateway @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Runtime image for a gateway built as a static musl binary. Shared by the +# gateway test suites, which each copy their own `dstack-gateway` next to their +# compose file before building. +FROM alpine:latest + +RUN apk add --no-cache \ + wireguard-tools \ + iproute2 \ + curl \ + ca-certificates + +COPY dstack-gateway /usr/local/bin/dstack-gateway + +RUN chmod +x /usr/local/bin/dstack-gateway && \ + mkdir -p /etc/gateway/certs /var/lib/gateway + +ENTRYPOINT ["/usr/local/bin/dstack-gateway", "-c", "/etc/gateway/gateway.toml"] diff --git a/dstack/gateway/test-run/TESTING.md b/dstack/gateway/test-run/TESTING.md index 650297e63..212870b67 100644 --- a/dstack/gateway/test-run/TESTING.md +++ b/dstack/gateway/test-run/TESTING.md @@ -2,242 +2,107 @@ # # SPDX-License-Identifier: Apache-2.0 -# Gateway test plan +# Gateway integration tests -This document records the local checks used for the gateway handshake-cache -change. The goal is to verify three things: +Three suites, all under docker compose, all driven from the host. None of them +needs root on the machine running them, and none of them leaves anything behind. -1. the generic `cached-cell` crate behaves correctly; -2. existing gateway control-plane and WaveKV flows still work; -3. the proxy data path does not call blocking `wg show latest-handshakes` per - request. +| Suite | Covers | Run it | +| --- | --- | --- | +| `e2e/` | certbot, ACME, DNS-01 and `dns-persist-01` | `e2e/run-e2e.sh` | +| `cluster/` | WaveKV replication, node identity, partition recovery, admin RPCs | `cluster/run-cluster-tests.sh` | +| `proxy-e2e/` | the proxy data path: splice, kTLS, half-close, idle reaping | `proxy-e2e/run-proxy-tests.sh` | -## Prerequisites +Each has a `--skip-build` flag that reuses the `dstack-gateway` binary already +staged next to its compose file, and a `down` subcommand that tears everything +down. -Run on Linux with: +## Attestation is on -- Rust toolchain; -- `sudo`, `ip`, `wg` / `wireguard-tools`; -- `curl`, `openssl`, `python3`; -- `wrk` for the performance test. +No suite turns the checks off. The gateways obtain their +RPC certificates from the guest agent simulator, which signs quotes under trust +anchors derived from `attestation/tee-simulator.json`; the mock collateral +service reconstructs the matching public roots from the same seed, and the +gateways verify each other through the normal production path. -The integration script creates temporary WireGuard interfaces named -`wavekv-test1`, `wavekv-test2`, and `wavekv-test3`, so it needs root privileges. +That is the point of the arrangement rather than a detail of it: the cluster's +mTLS is what `cluster/` exists to exercise, and a suite that switched the checks +off would be testing the switch. -## Unit and build checks +## One seed, one fixture per suite -From the repository root: +`attestation/fixture.yml` describes the simulator and the collateral service; +each suite brings up its own copy under its own `FIXTURE_NS`, and tears down +only that copy. -```bash -cargo test -p cached-cell -cargo test --manifest-path gateway/Cargo.toml -cargo check --manifest-path gateway/Cargo.toml -cargo clippy -- \ - -D warnings \ - -D clippy::expect_used \ - -D clippy::unwrap_used \ - --allow unused_variables -``` +The seed that signs the quotes and the seed that derives the verifying roots +must not drift apart, and they cannot: both come from files in `attestation/`, +so separate instances started from them agree by construction. Sharing one +running instance across the suites was the earlier arrangement and bought +nothing the files did not already guarantee -- while costing a teardown race, +where whichever suite finished first removed a fixture the others were still +using, and, in CI, three workflows that could not run at the same time. -Expected result: all commands pass. +## One project per test -## WaveKV / gateway integration test +The cluster suite gives each test a throwaway compose project of its own. Its +containers, its logs and its data directory are empty when it starts because +they are new -- not because something cleaned them up. -Build the gateway binary first: +That is the point rather than a detail. The version before it shared three +containers across all 28 tests and cleaned between them, which needed a wipe +helper, a time window on `docker logs`, and a lock to stop two runs from +clearing each other's state. Each of those three was the direct cause of a bug +during development, and each stops being possible here. -```bash -cargo build --release --manifest-path gateway/Cargo.toml -``` +## Unit and build checks -Then run the integration suite: +From `dstack/`: ```bash -cd gateway/test-run -sudo -E GATEWAY_BIN="$(pwd)/../../target/release/dstack-gateway" ./test_suite.sh -``` - -The suite starts real gateway processes and exercises: - -- CVM registration through `POST /prpc/RegisterCvm` on the debug service; -- admin RPCs such as `Admin.SetNodeUrl`, `Admin.SetNodeStatus`, and - `Admin.WaveKvStatus`; -- WaveKV persistent and ephemeral sync between gateway nodes; -- push propagation before the five-second periodic sync interval; -- periodic anti-entropy repair after a push is missed while a peer is offline; -- bootstrap recovery after a node loses its local WaveKV store while retaining - its node identity; -- convergence of divergent writes made on both sides of a partition; -- idempotence when opportunistic pushes overlap a periodic sync round; -- bootnode discovery retry, interrupted-sync recovery, and partial-cluster - bootstrap while another peer is unavailable; -- ephemeral-store convergence after a peer restart; -- node-ID conflict rejection followed by convergence under the replacement - node's fresh UUID; -- node restart, network partition recovery, periodic persistence, and node - up/down filtering. - -Expected result: - -```text -Tests passed: 28 +cargo test -p cached-cell +cargo test --manifest-path gateway/Cargo.toml +cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used \ + --allow unused_variables ``` -Important request paths covered by this suite: +## What runs in CI -| Path | Purpose | +| Workflow | Suite | | --- | --- | -| `POST /prpc/RegisterCvm` | Register a CVM, allocate a WireGuard IP, update gateway state. | -| `POST /prpc/Debug.Info` | Verify the debug service is available. | -| `POST /prpc/Debug.GetSyncData` | Inspect peer/node/instance data synced through WaveKV. | -| `POST /prpc/GetProxyState` | Compare in-memory proxy state with WaveKV state. | -| `POST /prpc/Admin.SetNodeUrl` | Register peer gateway URLs. | -| `POST /prpc/Admin.SetNodeStatus` | Mark nodes up/down and verify registration filtering. | -| `POST /prpc/Admin.WaveKvStatus` | Inspect WaveKV store status. | -| `POST /wavekv/sync/persistent` | Gateway-to-gateway persistent data sync. | -| `POST /wavekv/sync/ephemeral` | Gateway-to-gateway last-seen/handshake/connection sync. | -| `POST /wavekv/push/persistent` | Opportunistic persistent-state propagation. | -| `POST /wavekv/push/ephemeral` | Opportunistic ephemeral-state propagation. | - -## Real proxy data-path smoke test - -The integration suite above validates registration and sync, but it does not -open a client connection through the gateway proxy. For the proxy data path, use -this shape: - -1. Start one `dstack-gateway` with debug/admin enabled and `insecure_skip_attestation = true`. -2. Register a test CVM through the debug `RegisterCvm` RPC. -3. Bind a local HTTPS backend to the allocated CVM IP, for example - `10.0.51.2:23143`. -4. Serve a local DNS TXT response: - - ```text - _dstack-app-address.proxy-flow.local TXT "proxyflow:23143" - ``` - -5. Allow the backend port with `Admin.SetInstancePortPolicy` so the proxy data - path is not blocked by port-policy fail-close. -6. Send a request through the proxy: - - ```bash - curl -skf \ - --connect-to proxy-flow.local:13114:127.0.0.1:13114 \ - https://proxy-flow.local:13114/proxy-e2e - ``` - -Expected response from the backend: +| `.github/workflows/gateway-e2e-tests.yml` | `e2e/` | +| `.github/workflows/gateway-cluster-tests.yml` | `cluster/` | +| `.github/workflows/gateway-proxy-tests.yml` | `proxy-e2e/` | -```text -proxy-e2e-ok path=/proxy-e2e +```bash +gh pr checks --repo Dstack-TEE/dstack --watch=false ``` -Expected gateway log shape: +## The kTLS fallback arm -```text -got sni: proxy-flow.local -target address is proxyflow:23143 -connecting to 10.0.51.2:23143 -connected to 10.0.51.2:23143 -``` - -This confirms the real data flow: +One arm of the proxy suite asserts that a gateway configured for kTLS on a +kernel without the TLS ULP falls back to userspace instead of truncating a gated +transfer at the gate. It runs in its own container, because the condition is +produced by a seccomp profile -- `setsockopt(IPPROTO_TCP, TCP_ULP)` returns +`ENOPROTOOPT` -- and a seccomp profile is fixed when a container is created. -```text -client -> gateway proxy -> SNI parse -> DNS TXT lookup -> ProxyState selection -> backend TLS service -``` +See `proxy-e2e/README.md` for why that replaced taking the module away from the +host with `rmmod`. ## Proxy performance / hot-path check -The performance test uses the same real proxy data flow as the smoke test, with -one extra control: put a temporary `wg` wrapper earlier in `PATH` for the gateway -process. The wrapper delegates normal commands to `/usr/bin/wg`, but for - -```text -wg show latest-handshakes -``` - -it returns a fixed test public key and records the call. This verifies that the -proxy hot path does not execute blocking `wg show` for every request. - -Use `wrk` for three measurements: - -```bash -# Direct backend baseline. -wrk -t4 -c64 -d15s https://10.0.62.2:23243/bench - -# Gateway proxy with keep-alive. -wrk -t4 -c64 -d15s https://proxy-perf.local:13214/bench - -# Gateway proxy with new TLS connections. -wrk -t4 -c32 -d10s -H 'Connection: close' \ - https://proxy-perf.local:13214/bench-close -``` - -Reference result from the local PR run: +Not part of any suite, and not reproducible from this tree: the `wg` wrapper and +the `/bench` endpoints it describes were never checked in. The numbers below are +kept as the record of one manual run made for the handshake-cache change, and +the claim they support -- that the proxy hot path does not shell out to +`wg show latest-handshakes` per request -- is what the assertion in +`test_proxy.sh` (`test_accel_status`) covers on every run. ```text direct backend keep-alive: 71507 req/s, avg latency 1.14ms gateway proxy keep-alive: 33842 req/s, avg latency 8.83ms gateway proxy connection-close: 874 req/s, avg latency 33.45ms -``` -The same run handled more than 500k proxy keep-alive requests. The `wg` wrapper -recorded: - -```text -wg show latest-handshakes: 7 +wg show latest-handshakes: 7 (over 500k proxied keep-alive requests) wg syncconf: 3 ``` - -The important assertion is the call count: `wg show latest-handshakes` is only -used by startup/preload and the periodic refresh task, not once per proxied -request. - -## PR CI - -Check GitHub Actions before merging: - -```bash -gh pr checks --repo Dstack-TEE/dstack --watch=false -``` - -Expected result: all required checks pass, including `gateway`, `rust-checks`, -`prek`, `reuse-lint`, and CodeQL. - -## Proxy data-path integration tests - -`test_proxy.sh` runs a real gateway process and asserts on what reaches the -wire, across every combination of the two gated optimisations and both proxy -paths. It runs in CI (`.github/workflows/gateway-proxy-tests.yml`); unlike -`test_suite.sh` it needs no root for the gateway itself, only one `sudo ip link -add` for the link the gateway expects at startup. - -```bash -cd gateway/test-run -./test_proxy.sh # builds the gateway if needed -GATEWAY_BIN=../../target/release/dstack-gateway ./test_proxy.sh -BASE_PORT=39000 ./test_proxy.sh # if the default range is busy -KEEP_LOGS=1 ./test_proxy.sh # keep the work dir on success -``` - -What it covers: - -| group | asserts | -|---|---| -| data path | payloads survive byte-for-byte on both paths, under and over each gate, and under concurrency | -| close | the app closing reaches the client as an orderly TLS shutdown, including after kTLS offload | -| `timeouts.idle` | the idle watchdog reaps on every relay path -- buffered, spliced, kTLS -- and `data_timeout_enabled = false` still opts out | -| kTLS engagement | offload happens, and only once the gate fires; no TLS decrypt errors | -| capability fallbacks | a kernel without the TLS ULP warns and keeps serving untruncated; an inert `connection_rebalance` warns | -| runtime modes | every `thread_per_core` / `connection_rebalance` combination serves traffic | -| half-close | a client that finishes its request still gets the reply, on both paths and every arm | -| Status RPC | the accel counters reflect the traffic that ran | - -The suite adapts to the host: groups that need a capability the kernel or the -privileges do not provide are reported as `SKIP` with the reason, never silently -passed. - -Half-close needs a TLS client that can send `close_notify` without waiting for -the peer's, which `ssl.SSLSocket` cannot express: its only shutdown is -bidirectional, and dropping to `shutdown(SHUT_WR)` sends a bare FIN, which -mid-TLS is a truncation the peer is right to reject. `proxy/tlsclient.py` -drives the TLS state machine over memory BIOs to do it properly. diff --git a/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation new file mode 100644 index 000000000..416d73304 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src + +# The toolchain file first, on a layer of its own. +# +# The workspace pins a channel plus rustfmt, clippy, rust-analyzer and three +# extra targets, none of which the base image carries and none of which this +# build needs -- so the first cargo invocation makes rustup download and install +# a complete toolchain. Measured at ~27s, in each of the two fixture images, on +# every build: it happened inside the step below, which `COPY . .` invalidates +# on any change anywhere in the repo. Keyed on the toolchain file alone it is a +# cache hit until the pin actually moves. +COPY rust-toolchain.toml ./rust-toolchain.toml +RUN cargo --version + +# The cargo caches are BuildKit cache mounts, not image layers, because +# `COPY . .` is the whole repo and busts any layer after it on any source +# change. A mount survives that, and both fixture images name the same three, so +# whichever builds second reuses the first's artifacts. `sharing=locked` +# because cargo does not want two builds in one target directory; they +# serialise rather than corrupt. +# +# The binary is copied out inside the RUN: a cache mount is not part of the +# resulting layer, so a later `COPY --from=builder /src/dstack/target/...` +# would find nothing there. +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/src/dstack/target,sharing=locked \ + cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p mock-attestation \ + && mkdir -p /out && cp dstack/target/release/dstack-mock-attestation /out/ + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/dstack-mock-attestation /usr/local/bin/ +COPY dstack/gateway/test-run/attestation/tee-simulator.json /etc/dstack/tee-simulator.json +ENTRYPOINT ["/usr/local/bin/dstack-mock-attestation"] diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore similarity index 100% rename from dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore rename to dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore diff --git a/dstack/gateway/test-run/attestation/Dockerfile.simulator b/dstack/gateway/test-run/attestation/Dockerfile.simulator new file mode 100644 index 000000000..15a5b2c5b --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src + +# The toolchain file first, on a layer of its own. +# +# The workspace pins a channel plus rustfmt, clippy, rust-analyzer and three +# extra targets, none of which the base image carries and none of which this +# build needs -- so the first cargo invocation makes rustup download and install +# a complete toolchain. Measured at ~27s, in each of the two fixture images, on +# every build: it happened inside the step below, which `COPY . .` invalidates +# on any change anywhere in the repo. Keyed on the toolchain file alone it is a +# cache hit until the pin actually moves. +COPY rust-toolchain.toml ./rust-toolchain.toml +RUN cargo --version + +# The cargo caches are BuildKit cache mounts, not image layers, because +# `COPY . .` is the whole repo and busts any layer after it on any source +# change. A mount survives that, and both fixture images name the same three, so +# whichever builds second reuses the first's artifacts. `sharing=locked` +# because cargo does not want two builds in one target directory; they +# serialise rather than corrupt. +# +# The binary is copied out inside the RUN: a cache mount is not part of the +# resulting layer, so a later `COPY --from=builder /src/dstack/target/...` +# would find nothing there. +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/src/dstack/target,sharing=locked \ + cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p dstack-guest-agent-simulator \ + && mkdir -p /out && cp dstack/target/release/dstack-simulator /out/ + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /opt/dstack-simulator +COPY --from=builder /out/dstack-simulator /usr/local/bin/dstack-simulator +COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \ + sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./ +COPY dstack/gateway/test-run/attestation/simulator.toml ./simulator.toml +CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"] diff --git a/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore new file mode 100644 index 000000000..f53ffe346 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.git +**/target +**/node_modules +**/__pycache__ +**/.env +**/.env.* diff --git a/dstack/gateway/test-run/attestation/fixture.yml b/dstack/gateway/test-run/attestation/fixture.yml new file mode 100644 index 000000000..8c15efdaf --- /dev/null +++ b/dstack/gateway/test-run/attestation/fixture.yml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Shared attestation fixture for the gateway test suites. +# +# The guest agent simulator signs its quotes under trust anchors derived from +# `mock_attestation_seed`, and the collateral service reconstructs the matching +# public roots from the same seed. The two are only useful together and only +# while the seed matches, so they live in one file that every suite includes +# rather than being copied per suite, where they would drift apart and leave +# every peer quote failing to verify for no visible reason. +# +# Deliberately network-agnostic: an including file attaches these services to +# its own network. Compose merges that in while keeping what is defined here. + +services: + dstack-simulator: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.simulator + image: dstack-simulator:gateway-e2e + volumes: + - dstack-socket:/var/run/dstack + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] + interval: 1s + timeout: 1s + retries: 30 + + mock-attestation: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.mock-attestation + image: dstack-mock-attestation:gateway-e2e + # No container_name on purpose. A fixed one is global, so pinning it here + # would stop two suites -- which all include this file -- from being up at + # the same time. Compose still publishes `mock-attestation` as a network + # alias, so everything addressing it by name keeps working. + command: + - serve + - --listen + - 0.0.0.0:8088 + - --config + - /etc/dstack/tee-simulator.json + - --output + - /roots + volumes: + - attestation-roots:/roots + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8088/tpm/aia/root.pem"] + interval: 1s + timeout: 2s + retries: 30 + +# Named explicitly, and namespaced per suite. +# +# A per-test project has to reference these from outside, and compose's default +# `_` would make that a guess about which project happened to +# create them first -- so the names are pinned. `FIXTURE_NS` then keeps each +# suite's copy to itself. +# +# One instance shared by all three suites was the earlier arrangement, on the +# reasoning that the seed signing the quotes and the seed deriving the verifying +# roots must not drift apart. They cannot: both come from files in this +# directory, so three instances started from them agree by construction. What +# sharing the *running* instance bought instead was a teardown race -- whichever +# suite finished first tore down a fixture the others were still using -- and, +# in CI, three workflows that could not run at the same time. Neither is worth +# paying for something the files already guarantee. +networks: + attestation: + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + +volumes: + dstack-socket: + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots diff --git a/dstack/gateway/test-run/attestation/simulator.toml b/dstack/gateway/test-run/attestation/simulator.toml new file mode 100644 index 000000000..3f34c174e --- /dev/null +++ b/dstack/gateway/test-run/attestation/simulator.toml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +[default] +workers = 8 +max_blocking = 64 +ident = "dstack Gateway E2E Simulator" +temp_dir = "/tmp" +keep_alive = 10 +log_level = "info" + +[default.core] +keys_file = "/opt/dstack-simulator/appkeys.json" +compose_file = "/opt/dstack-simulator/app-compose.json" +sys_config_file = "/opt/dstack-simulator/sys-config.json" +data_disks = ["/"] + +[default.core.simulator] +attestation_file = "/opt/dstack-simulator/attestation.bin" +patch_report_data = true +# Re-sign the fixture quote under trust anchors derived from this seed instead +# of shipping the fixture's original, now-stale signature. The mock collateral +# service derives the matching roots from the same seed, so the gateways verify +# each other's quotes through the normal production path rather than skipping +# the check. Must stay in sync with the `mock_attestation_seed` in +# `attestation/tee-simulator.json`, which the collateral service reads. Two +# files, one value: the simulator config is TOML read by the simulator and the +# collateral config is JSON read by dstack-mock-attestation, so there is nowhere +# to put it once. They are kept adjacent, and the e2e run fails closed if they +# drift -- every peer quote stops verifying. +mock_attestation_seed = "4747474747474747474747474747474747474747474747474747474747474747" + +[internal] +address = "unix:/var/run/dstack/dstack.sock" +reuse = true diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json b/dstack/gateway/test-run/attestation/tee-simulator.json new file mode 100644 index 000000000..190eceab3 --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json @@ -0,0 +1,5 @@ +{ + "platform": "dstack-tdx", + "mock_attestation_seed": "4747474747474747474747474747474747474747474747474747474747474747", + "collateral_base_url": "http://mock-attestation:8088" +} diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json.license b/dstack/gateway/test-run/attestation/tee-simulator.json.license new file mode 100644 index 000000000..7ed638e2a --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: © 2026 Phala Network + +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/test-run/build-gateway-image.sh b/dstack/gateway/test-run/build-gateway-image.sh new file mode 100755 index 000000000..70cacaddc --- /dev/null +++ b/dstack/gateway/test-run/build-gateway-image.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Build the gateway as a static musl binary and wrap it in the alpine runtime +# image both test suites run. Shared so the two suites cannot end up testing +# different builds. +# +# Usage: build-gateway-image.sh [--skip-build] +# where to place the binary; also the docker build context + +set -e + +TEST_RUN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DSTACK="$(cd "$TEST_RUN_DIR/../.." && pwd)" + +DEST_DIR="${1:?usage: build-gateway-image.sh [--skip-build]}" +SKIP_BUILD="${2:-}" + +if [ "$SKIP_BUILD" != "--skip-build" ]; then + echo "[INFO] building dstack-gateway (musl static)..." >&2 + (cd "$REPO_DSTACK" && cargo build --release -p dstack-gateway \ + --target x86_64-unknown-linux-musl) + cp "$REPO_DSTACK/target/x86_64-unknown-linux-musl/release/dstack-gateway" "$DEST_DIR/" +fi + +if [ ! -f "$DEST_DIR/dstack-gateway" ]; then + echo "[ERROR] $DEST_DIR/dstack-gateway is missing; run without --skip-build" >&2 + exit 1 +fi + +echo "[INFO] building the gateway image..." >&2 +docker build -t dstack-gateway:test -f "$TEST_RUN_DIR/Dockerfile.gateway" "$DEST_DIR" diff --git a/dstack/gateway/test-run/cluster.sh b/dstack/gateway/test-run/cluster.sh deleted file mode 100755 index 27f58b4c6..000000000 --- a/dstack/gateway/test-run/cluster.sh +++ /dev/null @@ -1,441 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Gateway cluster management script for manual testing - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -GATEWAY_BIN="${SCRIPT_DIR}/../../target/release/dstack-gateway" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -TMUX_SESSION="gateway-cluster" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -show_help() { - echo "Gateway Cluster Management Script" - echo "" - echo "Usage: $0 " - echo "" - echo "Commands:" - echo " start Start a 3-node gateway cluster in tmux" - echo " stop Stop the cluster (keep tmux session)" - echo " reg Register a random instance" - echo " status Show cluster status" - echo " clean Destroy cluster and clean all data" - echo " attach Attach to tmux session" - echo " help Show this help" - echo "" -} - -# Generate certificates -generate_certs() { - mkdir -p "$CERTS_DIR" - mkdir -p "$RUN_DIR/certbot/live" - - # Generate CA certificate - if [[ ! -f "$CERTS_DIR/gateway-ca.key" ]]; then - log_info "Creating CA certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-ca.key" 2048 2>/dev/null - openssl req -x509 -new -nodes \ - -key "$CERTS_DIR/gateway-ca.key" \ - -sha256 -days 365 \ - -out "$CERTS_DIR/gateway-ca.cert" \ - -subj "/CN=Test CA/O=Gateway Test" \ - 2>/dev/null - fi - - # Generate RPC certificate signed by CA - if [[ ! -f "$CERTS_DIR/gateway-rpc.key" ]]; then - log_info "Creating RPC certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-rpc.key" 2048 2>/dev/null - openssl req -new \ - -key "$CERTS_DIR/gateway-rpc.key" \ - -out "$CERTS_DIR/gateway-rpc.csr" \ - -subj "/CN=localhost" \ - 2>/dev/null - cat > "$CERTS_DIR/ext.cnf" << EXTEOF -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 -EXTEOF - openssl x509 -req \ - -in "$CERTS_DIR/gateway-rpc.csr" \ - -CA "$CERTS_DIR/gateway-ca.cert" \ - -CAkey "$CERTS_DIR/gateway-ca.key" \ - -CAcreateserial \ - -out "$CERTS_DIR/gateway-rpc.cert" \ - -days 365 \ - -sha256 \ - -extfile "$CERTS_DIR/ext.cnf" \ - 2>/dev/null - rm -f "$CERTS_DIR/gateway-rpc.csr" "$CERTS_DIR/ext.cnf" - fi - - # Generate proxy certificates - local proxy_cert_dir="$RUN_DIR/certbot/live" - if [[ ! -f "$proxy_cert_dir/cert.pem" ]]; then - log_info "Creating proxy certificates..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$proxy_cert_dir/key.pem" \ - -out "$proxy_cert_dir/cert.pem" \ - -days 365 \ - -subj "/CN=localhost" \ - 2>/dev/null - fi - - # Generate unique WireGuard key pair for each node - for i in 1 2 3; do - if [[ ! -f "$CERTS_DIR/wg-node${i}.key" ]]; then - log_info "Generating WireGuard keys for node ${i}..." - wg genkey > "$CERTS_DIR/wg-node${i}.key" - wg pubkey < "$CERTS_DIR/wg-node${i}.key" > "$CERTS_DIR/wg-node${i}.pub" - fi - done -} - -# Generate node config -generate_config() { - local node_id=$1 - local rpc_port=$((13000 + node_id * 10 + 2)) - local wg_port=$((13000 + node_id * 10 + 3)) - local proxy_port=$((13000 + node_id * 10 + 4)) - local debug_port=$((13000 + node_id * 10 + 5)) - local admin_port=$((13000 + node_id * 10 + 6)) - local wg_ip="10.0.3${node_id}.1/24" - local other_nodes="" - local peer_urls="" - - # Read WireGuard keys for this node - local wg_private_key=$(cat "$CERTS_DIR/wg-node${node_id}.key") - local wg_public_key=$(cat "$CERTS_DIR/wg-node${node_id}.pub") - - for i in 1 2 3; do - if [[ $i -ne $node_id ]]; then - local peer_rpc_port=$((13000 + i * 10 + 2)) - if [[ -n "$other_nodes" ]]; then - other_nodes="$other_nodes, $i" - peer_urls="$peer_urls, \"$i:https://localhost:$peer_rpc_port\"" - else - other_nodes="$i" - peer_urls="\"$i:https://localhost:$peer_rpc_port\"" - fi - fi - done - - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - cat > "$RUN_DIR/node${node_id}.toml" << EOF -log_level = "info" -address = "0.0.0.0" -port = ${rpc_port} - -[tls] -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -rpc_domain = "" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -port = ${debug_port} -address = "127.0.0.1" - -[core.admin] -enabled = true -port = ${admin_port} -address = "127.0.0.1" - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_node${node_id}" - -[core.certbot] -enabled = false - -[core.wg] -private_key = "${wg_private_key}" -public_key = "${wg_public_key}" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.3${node_id}.1/31"] -client_ip_range = "10.0.3${node_id}.1/24" -config_path = "${RUN_DIR}/wg_node${node_id}.conf" -interface = "gw-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${RUN_DIR}/certbot/live/cert.pem" -cert_key = "${RUN_DIR}/certbot/live/key.pem" -base_domain = "test.local" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} - -[core.recycle] -enabled = true -interval = "30s" -timeout = "120s" -node_timeout = "300s" -EOF -} - -# Build gateway binary -build_gateway() { - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_info "Building gateway..." - (cd "$SCRIPT_DIR/.." && cargo build --release) - fi -} - -# Start cluster -cmd_start() { - build_gateway - generate_certs - - # Check if tmux session exists - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - log_warn "Cluster already running. Use 'clean' to restart." - cmd_status - return 0 - fi - - log_info "Generating configs..." - mkdir -p "$RUN_DIR" "$LOG_DIR" - for i in 1 2 3; do - generate_config $i - mkdir -p "$RUN_DIR/wavekv_node${i}" - done - - log_info "Starting cluster in tmux session '$TMUX_SESSION'..." - - # Create wrapper scripts that keep running even if gateway exits - for i in 1 2 3; do - cat > "$RUN_DIR/run_node${i}.sh" << RUNEOF -#!/bin/bash -cd "$SCRIPT_DIR" -while true; do - echo "Starting node ${i}..." - sudo RUST_LOG=info $GATEWAY_BIN -c $RUN_DIR/node${i}.toml 2>&1 | tee -a $LOG_DIR/node${i}.log - echo "Node ${i} exited. Press Ctrl+C to stop, or wait 3s to restart..." - sleep 3 -done -RUNEOF - chmod +x "$RUN_DIR/run_node${i}.sh" - done - - # Create tmux session - tmux new-session -d -s "$TMUX_SESSION" -n "node1" - tmux send-keys -t "$TMUX_SESSION:node1" "$RUN_DIR/run_node1.sh" Enter - - sleep 1 - - # Add windows for other nodes - tmux new-window -t "$TMUX_SESSION" -n "node2" - tmux send-keys -t "$TMUX_SESSION:node2" "$RUN_DIR/run_node2.sh" Enter - - tmux new-window -t "$TMUX_SESSION" -n "node3" - tmux send-keys -t "$TMUX_SESSION:node3" "$RUN_DIR/run_node3.sh" Enter - - # Add a shell window - tmux new-window -t "$TMUX_SESSION" -n "shell" - - sleep 3 - - log_info "Cluster started!" - echo "" - cmd_status - echo "" - log_info "Use '$0 attach' to view logs" -} - -# Stop cluster -cmd_stop() { - log_info "Stopping cluster..." - sudo pkill -9 -f "dstack-gateway.*node[123].toml" 2>/dev/null || true - sudo ip link delete gw-test1 2>/dev/null || true - sudo ip link delete gw-test2 2>/dev/null || true - sudo ip link delete gw-test3 2>/dev/null || true - log_info "Cluster stopped" -} - -# Clean everything -cmd_clean() { - cmd_stop - - # Kill tmux session - tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true - - log_info "Cleaning data..." - sudo rm -rf "$RUN_DIR/wavekv_node"* - sudo rm -f "$RUN_DIR/gateway-state-node"*.json - rm -f "$RUN_DIR/wg_node"*.conf - rm -f "$RUN_DIR/node"*.toml - rm -f "$RUN_DIR/run_node"*.sh - rm -rf "$LOG_DIR" - - log_info "Cleaned" -} - -# Show status -cmd_status() { - echo -e "${BLUE}=== Gateway Cluster Status ===${NC}" - echo "" - - for i in 1 2 3; do - local rpc_port=$((13000 + i * 10 + 2)) - local proxy_port=$((13000 + i * 10 + 4)) - local debug_port=$((13000 + i * 10 + 5)) - local admin_port=$((13000 + i * 10 + 6)) - - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - echo -e "Node $i: ${GREEN}RUNNING${NC}" - else - echo -e "Node $i: ${RED}STOPPED${NC}" - fi - echo " RPC: https://localhost:${rpc_port}" - echo " Proxy: https://localhost:${proxy_port}" - echo " Debug: http://localhost:${debug_port}" - echo " Admin: http://localhost:${admin_port}" - echo "" - done - - # Show instance count from first running node - for i in 1 2 3; do - local debug_port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - local response=$(curl -s -X POST "http://localhost:${debug_port}/prpc/GetSyncData" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null) - if [[ -n "$response" ]]; then - local n_instances=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('instances', [])))" 2>/dev/null || echo "?") - local n_nodes=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('nodes', [])))" 2>/dev/null || echo "?") - echo -e "${BLUE}Cluster State:${NC}" - echo " Nodes: $n_nodes" - echo " Instances: $n_instances" - fi - break - fi - done -} - -# Register a random instance -cmd_reg() { - # Find a running node - local debug_port="" - for i in 1 2 3; do - local port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - debug_port=$port - break - fi - done - - if [[ -z "$debug_port" ]]; then - log_error "No running nodes found. Start cluster first." - exit 1 - fi - - # Generate random WireGuard key pair - local private_key=$(wg genkey) - local public_key=$(echo "$private_key" | wg pubkey) - - # Generate random IDs - local app_id="app-$(openssl rand -hex 4)" - local instance_id="inst-$(openssl rand -hex 4)" - - log_info "Registering instance..." - log_info " App ID: $app_id" - log_info " Instance ID: $instance_id" - log_info " Public Key: $public_key" - - local response=$(curl -s \ - -X POST "http://localhost:${debug_port}/prpc/RegisterCvm" \ - -H "Content-Type: application/json" \ - -d "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" 2>/dev/null) - - if echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'wg' in d" 2>/dev/null; then - local client_ip=$(echo "$response" | python3 -c "import sys,json; print(json.load(sys.stdin)['wg']['client_ip'])" 2>/dev/null) - log_info "Registered successfully!" - echo -e " Client IP: ${GREEN}$client_ip${NC}" - echo "" - echo "Instance details:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - else - log_error "Registration failed:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - exit 1 - fi -} - -# Attach to tmux -cmd_attach() { - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - tmux attach -t "$TMUX_SESSION" - else - log_error "No cluster running" - exit 1 - fi -} - -# Main -case "${1:-help}" in - start) - cmd_start - ;; - stop) - cmd_stop - ;; - clean) - cmd_clean - ;; - status) - cmd_status - ;; - reg) - cmd_reg - ;; - attach) - cmd_attach - ;; - help|--help|-h) - show_help - ;; - *) - log_error "Unknown command: $1" - show_help - exit 1 - ;; -esac diff --git a/dstack/gateway/test-run/cluster/docker-compose.yml b/dstack/gateway/test-run/cluster/docker-compose.yml new file mode 100644 index 000000000..29d91ab1d --- /dev/null +++ b/dstack/gateway/test-run/cluster/docker-compose.yml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# WaveKV / cluster integration suite. +# +# Driven from the host: the driver rewrites a node's config and stops or starts +# its container between tests, which is what the ported tests need and what a +# runner inside the network could not do without the docker socket. +# +# Each node's config and data directory are bind mounts, not tmpfs: the tests +# read and write node state directly -- 17 of 28 do -- and four perform surgery +# on a stopped node's data directory (wipe it but keep node_uuid, stat the WAL). +# +# Host ports are left for docker to assign and the driver looks them up with +# `docker compose port`. Fixed host ports are a standing collision with whatever +# else the developer is running -- the suite this replaces squatted on 13012-13036 +# and carried a wait-for-the-port-to-free loop because of it. +# +# Config and data live under the test's own name, so a test cannot read or +# write what another one left. CURRENT_TEST is exported by the driver before it +# brings a project up; the default keeps `docker compose` usable by hand. +# +# The config is mounted as a DIRECTORY rather than a file. The driver rewrites +# node configs between tests, and a single-file bind mount pins the original +# inode, so a rewritten file would never be seen inside the container. + +# The fixture is a long-lived project of its own, started once by the driver. +# Each test then gets a throwaway project that joins the fixture's network, so a +# test's containers, logs and data start empty because they are new rather than +# because something cleaned them. +networks: + attestation: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + +volumes: + dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots + +services: + gateway-1: &node + image: ${GATEWAY_IMAGE:-dstack-gateway:test} + networks: + attestation: + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node1:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node1:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + environment: + - RUST_LOG=info,dstack_gateway=debug + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + cap_add: + - NET_ADMIN + + gateway-2: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node2:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node2:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + + gateway-3: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node3:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node3:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh new file mode 100644 index 000000000..320cd2e8b --- /dev/null +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -0,0 +1,350 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Host-side driver primitives for the gateway cluster suite. +# +# The tests need to stop a node, rewrite its config, wipe or doctor its data +# directory and start it again. All of that happens here, on the host, against +# bind mounts and `docker compose`. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="$SCRIPT_DIR/run" +CONFIG_DIR="$RUN_DIR/configs" +DATA_DIR="$RUN_DIR/data" +LOG_DIR="$RUN_DIR/logs" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { printf "${BLUE}[INFO]${NC} %s\n" "$1" >&2; } +log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1" >&2; } +log_error() { printf "${RED}[ERROR]${NC} %s\n" "$1" >&2; } +log_success() { printf "${GREEN}[PASS]${NC} %s\n" "$1" >&2; } +log_fail() { printf "${RED}[FAIL]${NC} %s\n" "$1" >&2; } + +# Host-side ports are assigned by docker, not fixed, so the suite cannot +# collide with whatever else is listening on the machine. Look them up once a +# container is running and cache the answer -- `docker compose port` is a +# process spawn, and the tests ask for these constantly. +declare -A _PORT_CACHE=() +container_port() { + local node_id=$1 + local container_port=$2 + # Keyed by project too: each test gets its own containers, so a port + # cached under a previous test's project points at something that no longer + # exists -- which reads as "the node did not come up". + local key="${CURRENT_TEST:-suite}:${node_id}:${container_port}" + local mapped + if [[ -n "${_PORT_CACHE[$key]:-}" ]]; then + echo "${_PORT_CACHE[$key]}"; return 0 + fi + mapped=$(compose port "gateway-${node_id}" "$container_port" 2>/dev/null | awk -F: 'NF{print $NF}') + [[ -n "$mapped" ]] || return 1 + _PORT_CACHE[$key]="$mapped" + echo "$mapped" +} + +# A restarted container gets a new host port, so anything that stops a node has +# to drop the cache or later lookups address a port nothing is listening on. +forget_ports() { + local node_id=$1 + local t="${CURRENT_TEST:-suite}" + unset '_PORT_CACHE['"$t:$node_id"':9012]' \ + '_PORT_CACHE['"$t:$node_id"':9015]' \ + '_PORT_CACHE['"$t:$node_id"':9016]' +} + +rpc_port() { container_port "$1" 9012; } +debug_port() { container_port "$1" 9015; } +admin_port() { container_port "$1" 9016; } + +# Write node $1's config, wiring its bootnode to $2 (empty for none). +# +# Everything except the bootnode is derived from the node id, which is why the +# original suite could regenerate this 30-odd times and only ever change one +# field. +generate_config() { + local node_id=$1 + local bootnode_url=${2:-""} + local dir="$CONFIG_DIR/${CURRENT_TEST:-suite}/node${node_id}" + mkdir -p "$dir" + cat >"$dir/gateway.toml" </dev/null +} + +# Each test runs in a compose project of its own, so its containers, its logs +# and its data start empty because they are new -- not because something cleaned +# them. CURRENT_TEST is set by the runner before each test. +compose() { + docker compose -p "cluster-${CURRENT_TEST:-suite}" \ + -f "$SCRIPT_DIR/docker-compose.yml" "$@" +} + +# The attestation fixture, namespaced to this suite. +# +# Each suite runs its own copy rather than sharing one project. The seed that +# signs quotes and the seed that derives the verifying roots come from files in +# `attestation/`, so separate instances agree by construction -- while a shared +# instance meant whichever suite finished first tore it down under the others, +# and meant the three CI workflows could not run at the same time. +FIXTURE_NS="dstack-fixture-cluster" +export FIXTURE_NS + +fixture_compose() { + docker compose -p "$FIXTURE_NS" \ + -f "$SCRIPT_DIR/../attestation/fixture.yml" "$@" +} + +# Every project this suite may have created, asked of the daemon rather than +# guessed. +# +# `compose down` with CURRENT_TEST unset addresses `cluster-suite`, which no +# test ever uses -- the runner sets CURRENT_TEST before the first one -- so the +# `down` subcommand tore down nothing and a killed run leaked one project per +# test, containers still attached to the shared attestation network. Label +# lookup finds them whatever they are called and whatever went wrong. +cluster_projects() { + # Matched on the compose file's absolute path, not on a `cluster-*` name + # pattern. The name is this suite's convention, not something the daemon + # guarantees, and a pattern wide enough to catch every per-test project is + # also wide enough to tear down an unrelated `cluster-…` project that + # happens to be on the same machine. `config_files` is set by compose to the + # file the container came from, so this selects exactly this checkout's + # containers -- another worktree running the same suite is left alone. + docker ps -a \ + --filter "label=com.docker.compose.project.config_files=$SCRIPT_DIR/docker-compose.yml" \ + --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \ + | sort -u +} + +down_all() { + local project + while read -r project; do + [ -n "$project" ] || continue + docker compose -p "$project" -f "$SCRIPT_DIR/docker-compose.yml" \ + down -v --remove-orphans >/dev/null 2>&1 || true + done < <(cluster_projects) +} + +# Clear the run tree's contents, from inside a container. +# +# The gateway runs as root and writes its certificates 0600 (see +# `safe_write_with_mode` in gateway/src/main.rs), so the host user cannot delete +# what a previous run left. Node state is a bind mount, which `compose down -v` +# does not touch either -- so without this every test resumed from the previous +# run's store. Test names are fixed, so the stale directory is always the one +# the test is about to use: `test_cross_node_data_sync` found its instance +# already replicated and passed with sync entirely broken, and +# `test_partial_cluster_bootstrap` never had to bootstrap. Only the first run on +# a fresh checkout -- which is what CI does, and what this was verified on -- +# behaved as the comments describe. +# +# Contents only. `run` is what the suite's lock is held on, and replacing the +# directory would detach that lock exactly the way a deleted lock file does. +wipe_run_tree() { + mkdir -p "$RUN_DIR" + docker run --rm -v "$RUN_DIR:/r" alpine:latest \ + find /r -mindepth 1 -delete >/dev/null 2>&1 \ + || log_warn "could not clear $RUN_DIR; a stale store may make assertions vacuous" +} + +# Discard whatever the previous test used and start this one from nothing. +new_test_project() { + compose down -v --remove-orphans >/dev/null 2>&1 || true + _PORT_CACHE=() + CURRENT_TEST="$1" + export CURRENT_TEST + mkdir -p "$DATA_DIR/$CURRENT_TEST" +} + +start_node() { + local node_id=$1 + # `compose start` exits 0 when the service has no container yet -- it prints + # "has no container to start" and succeeds -- so an `||` fallback to + # `compose up -d` never fires. Under one project per test the container + # usually does not exist, which made every node after the first look like it + # failed to come up. `up -d` both creates and starts, and is a no-op for a + # container that is already running. + compose up -d "gateway-${node_id}" >/dev/null 2>&1 || + compose start "gateway-${node_id}" >/dev/null 2>&1 + forget_ports "$node_id" + wait_for_debug "$node_id" 60 || { log_error "node ${node_id} did not come up"; return 1; } +} + +stop_node() { + local node_id=$1 + compose stop -t 5 "gateway-${node_id}" >/dev/null 2>&1 || true + forget_ports "$node_id" +} + +wait_for_debug() { + local node_id=$1 + local timeout=${2:-60} + # Wall clock, not an iteration count: each pass forks `docker compose port` + # and a curl, so counting `sleep 1`s overshot the stated timeout by half + # again. See `wait_until` in rpc.sh for the same fix on the sync waits. + local deadline=$((SECONDS + timeout)) + local port + while [ "$SECONDS" -lt "$deadline" ]; do + # Resolved inside the loop on purpose: docker only reports the mapping + # once the container is running, so looking it up once up front would + # leave every later request aimed at an empty port. + if port=$(debug_port "$node_id") && [ -n "$port" ] && + curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.Info" \ + -H 'Content-Type: application/json' -d '{}' >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +# Two ported tests assert on what a node logged. Containers keep that in the +# daemon rather than a file, so materialise it on demand and leave the +# assertions themselves unchanged. +# +# No window parameter, and none needed: see the note in the body. +dump_log() { + local node_id=$1 + mkdir -p "$LOG_DIR" + # No time window. This container was created for this test, so everything in + # its log belongs to this test -- and the window the shared-container design + # needed is what made the docker-logs timezone bug possible. + compose logs --no-color "gateway-${node_id}" \ + >"$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" 2>&1 || true + echo "$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" +} + + +node_data_dir() { echo "$DATA_DIR/${CURRENT_TEST:-suite}/node$1"; } + +# The gateway runs as root inside its container, so everything it writes into +# the bind-mounted data directory is root-owned. Reading it from the host is +# fine -- node_uuid and the WAL are world-readable, which is all the surgery +# tests need -- but removing or replacing it is not. Do that in a throwaway +# root container rather than reaching for sudo, which is exactly the dependency +# this port exists to shed. +# +# $1 is a shell snippet run against /data, the parent of the per-node dirs. +data_op() { + docker run --rm -v "$DATA_DIR/${CURRENT_TEST}:/data" alpine:latest sh -c "$1" +} + +# Wipe a stopped node's store but let it keep the identity it already published, +# which is what a node looks like after losing its disk but not its config. +# Fails loudly. The `&&` after the first `cp` short-circuits the wipe if +# node_uuid is not there, and errexit is off inside a test body (they run as the +# condition of an `if`), so an unchecked call let +# `test_bootstrap_after_data_dir_loss` "recover" a store that was never lost. +wipe_data_keeping_uuid() { + local node_id=$1 + data_op "set -e + cp /data/node${node_id}/wavekv/node_uuid /tmp/uuid + rm -rf /data/node${node_id}/wavekv/* /data/node${node_id}/wavekv/.[!.]* 2>/dev/null || true + cp /tmp/uuid /data/node${node_id}/wavekv/node_uuid" \ + || { log_error "could not wipe node ${node_id}'s store; the test below would prove nothing"; return 1; } +} + +# Wipe a stopped node completely, identity included -- a stranger turning up +# with a node id that is already taken. Contents only: the directory itself is a +# bind-mount source, and replacing it swaps the inode the mount was set against. +wipe_data() { + local node_id=$1 + # Same reasoning as `wipe_data_keeping_uuid`: a silently skipped wipe leaves + # the following assertions testing the wrong starting state. The globs are + # allowed to match nothing; the operation as a whole is not allowed to fail. + data_op "rm -rf /data/node${node_id}/..?* /data/node${node_id}/.[!.]* /data/node${node_id}/* 2>/dev/null; exit 0" \ + || { log_error "could not wipe node ${node_id}'s data directory"; return 1; } +} diff --git a/dstack/gateway/test-run/cluster/rpc.sh b/dstack/gateway/test-run/cluster/rpc.sh new file mode 100644 index 000000000..8a870f97b --- /dev/null +++ b/dstack/gateway/test-run/cluster/rpc.sh @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# RPC helpers for the cluster suite. +# +# These take a node id rather than a port. The process-based suite passed ports +# around because they were fixed and computable; here docker assigns them, so +# resolving one is a lookup and every caller would otherwise have to do it. + +debug_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(debug_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +admin_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(admin_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +json_field() { python3 -c "$1" 2>/dev/null; } + +check_debug_service() { + debug_call "$1" Debug.Info \ + | json_field "import sys,json; d=json.load(sys.stdin); assert 'base_domain' in d" +} + +debug_get_sync_data() { debug_call "$1" Debug.GetSyncData; } + +# A WireGuard public key is 32 bytes; derive a distinct one per seed so +# registrations do not collide. +test_public_key() { + python3 -c "import base64; print(base64.b64encode(int($1).to_bytes(32, 'big')).decode())" +} + +debug_register_cvm() { + local node_id=$1 + local public_key=$2 + local app_id=${3:-testapp} + local instance_id=${4:-testinstance} + debug_call "$node_id" RegisterCvm \ + "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" +} + +# Prints the allocated client_ip, or fails. +verify_register_response() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) + assert 'wg' in d, 'missing wg config' + assert 'client_ip' in d['wg'], 'missing client_ip' + print(d['wg']['client_ip']) +except Exception as e: + print(f'ERROR: {e}', file=sys.stderr) + sys.exit(1) +" 2>/dev/null +} + +_sync_data_has() { + local node_id=$1 + local collection=$2 + local peer_node_id=$3 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for entry in d.get('$collection', []): + if entry.get('node_id') == $peer_node_id: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} + +has_peer_addr() { _sync_data_has "$1" peer_addrs "$2"; } +has_node_info() { _sync_data_has "$1" nodes "$2"; } + +get_n_instances() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +get_n_nodes() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('nodes', []))) +except Exception: + print(0) +" 2>/dev/null +} + +# Two different endpoints, and conflating them silently breaks whatever reads +# the wrong one: WaveKvStatus carries the store digests and key counts, Status +# carries node identity and the peer list. +admin_wavekv_status() { admin_call "$1" Admin.WaveKvStatus; } +admin_status() { admin_call "$1" Admin.Status; } + +get_store_digest() { + local node_id=$1 + local store=$2 + admin_wavekv_status "$node_id" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['$store']['digest'])" 2>/dev/null || true +} + +# A node's own uuid, as it reports itself in its node list. +get_node_uuid() { + local node_id=$1 + admin_status "$node_id" | python3 -c " +import sys, json +d = json.load(sys.stdin) +me = d.get('id') +for n in d.get('nodes', []): + if n.get('id') == me: + print(n.get('uuid', '')) + break +" 2>/dev/null || true +} + +admin_set_node_url() { + admin_call "$1" Admin.SetNodeUrl "{\"id\": $2, \"url\": \"$3\"}" +} + +# Peers address each other by container hostname, not by a published port: the +# gateways talk over the compose network, and only the driver goes through the +# host. +# Wait until every node in the set has both a peer address and a node record for +# every other. +# +# `setup_peers` only pushes SetNodeUrl; the nodes then have to reach each other +# and record it, which is what the tests were sleeping a flat 4-20s for. Sleeping +# is both slower and weaker: a blind wait cannot tell "peering formed in 800ms" +# from "peering never formed and the assertion below is about to test something +# else". +# +# Both collections, not just `peer_addrs`. They settle at different times: +# `peer_addrs` is written when SetNodeUrl lands, while `nodes` needs an actual +# sync round to carry the record across. Waiting on the first alone returns +# earlier than the blind settle it replaced did, which is how +# `test_multi_node_sync` -- whose own assertions cover both -- started failing on +# `node_info` while `peer_addr` was already there. `nodes` implies a completed +# round, so it is the honest reading of "the nodes found each other" and it is +# what every caller here is really waiting for. +wait_for_peers() { + local timeout_seconds=$1; shift + wait_until "$timeout_seconds" _all_peers_known "$@" +} + +_all_peers_known() { + local node_ids=("$@") src dst + for src in "${node_ids[@]}"; do + for dst in "${node_ids[@]}"; do + [ "$src" = "$dst" ] && continue + has_peer_addr "$src" "$dst" || return 1 + has_node_info "$src" "$dst" || return 1 + done + done + return 0 +} + +setup_peers() { + local node_ids=("$@") + local src dst + for src in "${node_ids[@]}"; do + for dst in "${node_ids[@]}"; do + [ "$src" = "$dst" ] && continue + admin_set_node_url "$src" "$dst" "https://gateway-${dst}:9012" >/dev/null + done + done +} + +# Wall-clock deadline, not an iteration count. +# +# These loops used to run `$((timeout_seconds * 10))` iterations of "probe, then +# sleep 0.1", on the assumption that an iteration costs 0.1s. Every probe forks +# a curl and a python3, so an iteration costs closer to 0.15s and the real +# window was ~1.5x what the caller asked for. That is not a rounding error here: +# `test_push_fast_path` passes 3 to prove a push arrived *before* the 5s +# periodic sync could have done it anyway, and the true window was ~4.8s -- so a +# completely dead push path was caught by the periodic round and the test still +# went green. Bound on time and the number means what it says. +wait_until() { + local timeout_seconds=$1; shift + local deadline=$((SECONDS + timeout_seconds)) + while :; do + "$@" && return 0 + [ "$SECONDS" -lt "$deadline" ] || return 1 + sleep 0.1 + done +} + +_has_n_instances() { + [ "$(get_n_instances "$1")" -ge "$2" ] 2>/dev/null +} + +wait_for_instances() { + local node_id=$1 + local expected=$2 + local timeout_seconds=$3 + wait_until "$timeout_seconds" _has_n_instances "$node_id" "$expected" +} + +_digests_match() { + local d1 d2 + d1=$(get_store_digest "$2" "$1") + d2=$(get_store_digest "$3" "$1") + [ -n "$d1" ] && [ "$d1" = "$d2" ] +} + +wait_for_digest_match() { + local store=$1 + local node_a=$2 + local node_b=$3 + local timeout_seconds=$4 + wait_until "$timeout_seconds" _digests_match "$store" "$node_a" "$node_b" +} + +get_n_keys() { + admin_wavekv_status "$1" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['persistent']['n_keys'])" 2>/dev/null || echo 0 +} + +# The gateway writes its CA where only root can read it, so copy it out for the +# one test that validates the RPC chain rather than skipping verification. +export_ca_cert() { + local node_id=$1 + local dest="$RUN_DIR/node${node_id}-ca.cert" + data_op "cat /data/node${node_id}/certs/gateway-ca.cert" >"$dest" 2>/dev/null + [ -s "$dest" ] || return 1 + echo "$dest" +} + +get_n_peer_addrs() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('peer_addrs', []))) +except Exception: + print(0) +" 2>/dev/null +} + +debug_get_proxy_state() { debug_call "$1" GetProxyState; } + +get_n_proxy_state_instances() { + debug_get_proxy_state "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +admin_set_node_status() { + admin_call "$1" Admin.SetNodeStatus "{\"id\": $2, \"status\": \"$3\"}" +} + +get_peer_url_from_sync() { + local node_id=$1 + local peer_node_id=$2 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for pa in d.get('peer_addrs', []): + if pa.get('node_id') == $peer_node_id: + print(pa.get('url', '')) + sys.exit(0) + print('') +except Exception: + print('') +" 2>/dev/null +} + +# Whether a RegisterCvm response offered the CVM a given gateway. +response_lists_gateway() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for gw in d.get('gateways', []): + if gw.get('id') == $2: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh new file mode 100755 index 000000000..61d79ee3e --- /dev/null +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -0,0 +1,259 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Gateway cluster (WaveKV) integration suite, driven from the host. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +# shellcheck source=rpc.sh +source "$SCRIPT_DIR/rpc.sh" +# shellcheck source=tests.sh +source "$SCRIPT_DIR/tests.sh" + +SKIP_BUILD="" +KEEP_RUNNING=false +ONLY="" +while [[ $# -gt 0 ]]; do + case $1 in + --skip-build) SKIP_BUILD="--skip-build"; shift ;; + --keep-running) KEEP_RUNNING=true; shift ;; + --only) ONLY="$2"; shift 2 ;; + down) + # Every per-test project, not just the one CURRENT_TEST happens to + # name, plus the shared fixture -- otherwise CI's teardown leaves + # the fixture project and its global network and volumes behind on + # any runner that outlives the job. + down_all + fixture_compose down -v --remove-orphans >/dev/null 2>&1 || true + exit 0 ;; + -h|--help) + echo "Usage: $0 [--skip-build] [--keep-running] [--only ] | down" + exit 0 ;; + *) log_error "unknown option: $1"; exit 1 ;; + esac +done + +# Two runs of this suite share one compose project, one set of container names +# and one data directory, so a second run stops the containers the first is +# using and wipes the store out from under it. The result does not look like a +# collision -- it looks like flaky cross-node sync, because a node's writes +# vanish mid-test. Refuse to start instead. +# The lock is taken on the run directory itself, not on a file inside it. +# Deleting a lock FILE does not release the lock -- it detaches it: the holder +# keeps its inode while the next run creates a fresh one and flocks that +# successfully. Three runs of this suite overlapped that way, each wiping the +# others' state, and the failures read as flaky cross-node sync rather than as +# a collision. A directory is not something a cleanup step removes by name, and +# `rm -rf run/data run/configs run/logs` leaves it in place. +mkdir -p "$RUN_DIR" +exec 9<"$RUN_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($RUN_DIR)" + exit 1 +fi + +# No process-name guard beyond the lock. One was tried and false-positived on +# its own first run: `$(...)` forks a subshell whose argv is identical to the +# script's, so `pgrep -f` counted the check itself as a second instance. The +# directory lock above is the correct primitive -- it is race-free and needs no +# pattern matching. + +# Probed BEFORE the trap is installed, not next to the `up` that follows it. +# +# The fixture is shared with the other two suites and is meant to outlive any +# one of them, so cleanup tears it down only if this run started it. Reading +# that after `trap cleanup EXIT` meant every failure in between -- an image +# build, a wipe, a `docker` that is not there -- ran cleanup with the variable +# unset, took the `:-0` default, concluded it had started the fixture, and +# removed a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +cleanup() { + $KEEP_RUNNING && return 0 + # The last test's project is still up; the fixture outlives them all. + compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi +} +trap cleanup EXIT + +TESTS_PASSED=0 +TESTS_FAILED=0 +check() { + local name="$1"; shift + if "$@"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + fi +} + +# ---------------------------------------------------------------- assertions + +# Every node must see all three, on every node. +cluster_has_three_nodes() { + local node_id=$1 port count + port=$(debug_port "$node_id") + count=$(curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.GetSyncData" \ + -H 'Content-Type: application/json' -d '{}' 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin).get("nodes",[])))' 2>/dev/null) || return 1 + [ "$count" = "3" ] +} + +# Positive evidence that the cluster's mTLS is real: the certificate a node +# serves carries the app_id extension its peers pin, which only happens when it +# was issued through the guest agent against a verifiable quote. Absence of +# errors would not prove this -- a validator that never ran is also silent. +cert_carries_app_id() { + local node_id=$1 + data_op "cat /data/node${node_id}/certs/gateway-rpc.cert" 2>/dev/null \ + | openssl x509 -noout -text 2>/dev/null \ + | grep -q "1.3.6.1.4.1.62397.1.3" +} + +# And the negative: no peer was ever turned away. +# +# Deliberately not matching "bootnode discovery retry failed". A node whose +# bootnode is not listening yet retries and succeeds, which is a startup race, +# not an authentication failure -- treating it as one made this assertion fail +# on whichever node happened to start first. Convergence is what proves the +# retry worked, and that is asserted separately. +no_peer_was_rejected() { + local node_id=$1 log + log=$(dump_log "$node_id") + # `dump_log` ends in `|| true`, and `! grep -q` on an empty file is true, so + # without this the assertion passed whenever the log could not be collected + # at all -- the failure mode a negative assertion is most exposed to. Prove + # there is something to have searched first. + [ -s "$log" ] || { log_error "node ${node_id} produced no log to check"; return 1; } + ! grep -qE "does not contain app_id|invalid quote|app_id mismatch" "$log" +} + +# ---------------------------------------------------------------- main + +log_info "==========================================" +log_info "dstack-gateway cluster suite" +log_info "==========================================" + +# Discard anything a previous run left: node state is a bind mount that +# `compose down -v` does not remove and the host user cannot delete. +down_all +wipe_run_tree +mkdir -p "$DATA_DIR" + +log_info "starting the attestation fixture" +fixture_compose build >/dev/null +fixture_compose up -d --wait >/dev/null + +"$SCRIPT_DIR/../build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD + +log_info "building the suite image" +compose build >/dev/null + +# The smoke checks get a project of their own for the same reason every test +# does: the first test's new_test_project tears down whatever came before it, +# and without this the smoke nodes were what it tore down. +# +# It has to come before the configs are written: config and data paths are named +# after CURRENT_TEST, so generating them first puts them where nothing looks. +new_test_project smoke + +log_info "generating node configs" +generate_config 1 "" +generate_config 2 "https://gateway-1:9012" +generate_config 3 "https://gateway-1:9012" + +log_info "starting the cluster" +compose up -d >/dev/null + +for n in 1 2 3; do + check "node $n debug service is up" wait_for_debug "$n" 90 +done + +log_info "waiting for the cluster to converge" +converged=false +for _ in $(seq 1 30); do + if cluster_has_three_nodes 1 && cluster_has_three_nodes 2 && cluster_has_three_nodes 3; then + converged=true; break + fi + sleep 2 +done +$converged || log_warn "cluster did not converge within 60s" + +for n in 1 2 3; do + check "node $n sees all three nodes" cluster_has_three_nodes "$n" + check "node $n serves a certificate carrying its app_id" cert_carries_app_id "$n" + check "node $n rejected no peer" no_peer_was_rejected "$n" +done + +# ------------------------------------------------------------- ported tests +# +# Every row here must also be ticked in the port checklist only once it has been +# seen to FAIL against a broken cluster. Green on its own proves nothing. +run_ported() { + local name="$1" + log_info "---------- $name ----------" + new_test_project "$name" + if "$name"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + # Under one project per test the next test's new_test_project tears this + # one down, so a failure's container logs are gone before anyone can + # read them. Dumping them here is what makes a failure diagnosable at + # all: test_partial_cluster_bootstrap failed with nothing in the suite + # log but its own config lines, and there was no way to tell which step + # had failed. An explicit `if` rather than `A && B`, because SC2015 has + # already made a successful run exit non-zero once. + for _n in 1 2 3; do + if compose ps -q "gateway-${_n}" 2>/dev/null | grep -q .; then + log_info " saved log: $(dump_log "$_n")" + fi + done + fi +} + +ALL_TESTS=(test_persistence test_status_endpoint test_prpc_register \ + test_prpc_info test_wal_integrity \ + test_multi_node_sync test_node_recovery test_cross_node_data_sync \ + test_push_fast_path test_periodic_repair_after_missed_push \ + test_bootstrap_after_data_dir_loss test_divergent_partition_writes \ + test_push_periodic_overlap test_delayed_bootnode_recovery \ + test_interrupted_sync_recovery test_ephemeral_recovery \ + test_partial_cluster_bootstrap test_node_id_reuse_rejected \ + test_client_registration_persistence test_stress_writes \ + test_network_partition test_three_node_cluster \ + test_three_node_bootnode test_periodic_persistence \ + test_admin_set_node_url test_admin_set_node_status \ + test_node_status_register_exclude test_node_status_register_reject) + +# An unknown --only name used to select nothing, leave TESTS_FAILED at 0 and +# exit 0: a green run of zero tests, which is the one result this suite must +# never produce. +if [ -n "$ONLY" ]; then + printf '%s\n' "${ALL_TESTS[@]}" | grep -qx -- "$ONLY" || { + log_error "unknown test: $ONLY" + exit 1 + } +fi + +for t in "${ALL_TESTS[@]}"; do + [ -n "$ONLY" ] && [ "$t" != "$ONLY" ] && continue + run_ported "$t" +done + +log_info "==========================================" +log_info "Passed: $TESTS_PASSED" +log_info "Failed: $TESTS_FAILED" +log_info "==========================================" +[ "$TESTS_FAILED" -eq 0 ] diff --git a/dstack/gateway/test-run/cluster/tests.sh b/dstack/gateway/test-run/cluster/tests.sh new file mode 100644 index 000000000..da6c8cbd3 --- /dev/null +++ b/dstack/gateway/test-run/cluster/tests.sh @@ -0,0 +1,903 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Ported from test-run/test_suite.sh. Each test keeps the assertions it had; +# what changed is how nodes are started, stopped and reset. + +# Nothing to clean: the runner gave this test a compose project of its own, so +# its containers and its data directory are new. Kept as a no-op so each test +# still reads as declaring where it starts from. +cleanup_cluster() { :; } + +# ------------------------------------------------------------------ quick + +test_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + sleep 2 + local keys_after_write + keys_after_write=$(get_n_keys 1) + log_info "keys after startup: $keys_after_write" + + stop_node 1 + start_node 1 || return 1 + + local keys_after_restart + keys_after_restart=$(get_n_keys 1) + log_info "keys after restart: $keys_after_restart" + + # get_n_keys ends in `|| echo 0`, so an unreachable node reports zero keys + # rather than failing. Without this guard a node that never wrote anything + # gives 0 >= 0 and the test passes having proved nothing -- the same shape + # as the empty-store digest match that made test_ephemeral_recovery hollow. + [ "${keys_after_write:-0}" -gt 0 ] 2>/dev/null || { + log_error "nothing was in the store before the restart, so surviving it proves nothing: keys=${keys_after_write:-0}" + return 1; } + if [ "$keys_after_restart" -ge "$keys_after_write" ] 2>/dev/null; then + return 0 + fi + log_error "expected >= $keys_after_write keys, got $keys_after_restart" + return 1 +} + +test_status_endpoint() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + admin_wavekv_status 1 | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d['enabled'] is True, 'enabled should be True' +assert 'persistent' in d, 'missing persistent' +assert 'ephemeral' in d, 'missing ephemeral' +assert d['persistent']['wal_enabled'] is True, 'persistent wal should be enabled' +assert d['ephemeral']['wal_enabled'] is False, 'ephemeral wal should be disabled' +assert 'peers' in d['persistent'], 'missing peers in persistent' +" +} + +test_prpc_register() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 501)" deadbeef cafebabe) + verify_register_response "$response" >/dev/null || { + log_error "RegisterCvm did not return a client_ip"; return 1; } +} + +test_prpc_info() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + local ca port + ca=$(export_ca_cert 1) || { log_error "could not read the node's CA certificate"; return 1; } + port=$(rpc_port 1) || return 1 + + curl -s --cacert "$ca" --resolve "gateway-1:${port}:127.0.0.1" \ + -X POST "https://gateway-1:${port}/prpc/Info" \ + -H "Content-Type: application/json" -d '{}' 2>/dev/null | python3 -c " +import sys, json +d = json.load(sys.stdin) +if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) +assert 'base_domain' in d, 'missing base_domain' +assert 'external_port' in d, 'missing external_port' +" +} + +test_wal_integrity() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key response success=0 + for i in $(seq 1 5); do + key=$(test_public_key $((800 + i))) + response=$(debug_register_cvm 1 "$key" "wal_app$i" "wal_inst$i") + verify_register_response "$response" >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 5 ] || { log_error "registered only $success/5 clients"; return 1; } + + # The WAL has to exist and be non-empty: a store that silently stopped + # journalling would still answer every RPC above. + local wal_size + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + log_info "WAL size: ${wal_size} bytes" + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null +} + +# ------------------------------------------------------------------- sync + +# Two nodes must learn each other's address and node record, in both directions. +test_multi_node_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local ok=0 + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; ok=1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; ok=1; } + has_node_info 1 2 || { log_error "node 1 missing node_info for node 2"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1"; ok=1; } + return $ok +} + +# A node that was down has to catch up once it is back. +test_node_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + sleep 3 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local ok=0 + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1 after recovery"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1 after recovery"; ok=1; } + return $ok +} + +# A CVM registered on one node must appear on the other, and each node's two +# views of it -- the replicated store and the proxy's own state -- must agree. +test_cross_node_data_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + local client_ip + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" app1 inst1)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + # Both stores on both nodes, then read all four for the equality checks + # below. `|| true` because the assertions are what report -- reaching the + # ceiling should produce the counts that explain it, not a bare timeout. + wait_until 20 _both_nodes_hold_an_instance || true + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + return $ok +} + +_node1_logged_a_periodic_persist() { + grep -q "periodic persist completed" "$(dump_log 1)" +} + +_both_nodes_hold_an_instance() { + local n kv ps + for n in 1 2; do + kv=$(get_n_instances "$n") + ps=$(get_n_proxy_state_instances "$n") + # Same reasoning as `_three_node_views_converged`: the caller asserts + # `kv == ps` as well, so the wait has to cover it. + [ "${kv:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${ps:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${kv:-0}" -eq "${ps:-1}" ] 2>/dev/null || return 1 + done + return 0 +} + +# An opportunistic push must land well inside the 5s periodic interval. +test_push_fast_path() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before + before=$(get_n_instances 2) + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" push_app push_instance)" \ + >/dev/null || return 1 + + # 3s < the 5s periodic interval, so arriving in time is what proves the push + # path ran rather than the anti-entropy round that would have caught it anyway. + wait_for_instances 2 $((before + 1)) 3 || { + log_error "node 2 did not receive the write before the periodic interval"; return 1; } + wait_for_digest_match persistent 1 2 3 || { + log_error "persistent digests did not converge after push"; return 1; } +} + +# A write made while a peer was down must still reach it, via the periodic round. +test_periodic_repair_after_missed_push() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before + before=$(get_n_instances 2) + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" repair_app repair_instance)" \ + >/dev/null || return 1 + sleep 1 + start_node 2 || return 1 + setup_peers 1 2 + + wait_for_instances 2 $((before + 1)) 15 || { + log_error "periodic sync did not repair the missed write"; return 1; } +} + +# Losing the local store must not cost the node its identity, and it must +# refill from the peer rather than come back empty. +test_bootstrap_after_data_dir_loss() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" bootstrap_app bootstrap_instance)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || return 1 + + local old_uuid new_uuid + old_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ "$old_uuid" != "null" ]; } || { + log_error "node 2 did not report its identity before recovery"; return 1; } + + stop_node 2 + wipe_data_keeping_uuid 2 || return 1 + start_node 2 || return 1 + + wait_for_instances 2 1 15 || { + log_error "node 2 did not bootstrap after losing its local store"; return 1; } + # get_store_digest ends in `|| true`, so an unreachable node yields an empty + # string and empty equals empty. wait_for_instances above speaks for node 2; + # nothing has spoken for node 1, so check both are non-empty before + # concluding anything from their being equal. + local d1 d2 + d1=$(get_store_digest 1 persistent); d2=$(get_store_digest 2 persistent) + { [ -n "$d1" ] && [ -n "$d2" ]; } || { + log_error "could not read both digests, so comparing them proves nothing: node1='${d1}' node2='${d2}'" + return 1; } + [ "$d1" = "$d2" ] || { + log_error "persistent digests differ after bootstrap: node1='${d1}' node2='${d2}'"; return 1; } + + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + new_uuid=$(get_node_uuid 2) + { [ -n "$new_uuid" ] && [ "$new_uuid" != "null" ]; } || { + log_error "node 2 did not report its post-recovery identity"; return 1; } + [ "$old_uuid" = "$new_uuid" ] || { + log_error "losing the WaveKV store unexpectedly changed the node UUID"; return 1; } +} + +# Writes made on both sides of a partition must both survive the merge. +test_divergent_partition_writes() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" left_app left_instance)" \ + >/dev/null || return 1 + + stop_node 1 + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA=" right_app right_instance)" \ + >/dev/null || return 1 + + start_node 1 || return 1 + setup_peers 1 2 + { wait_for_instances 1 2 15 && wait_for_instances 2 2 15; } || { + log_error "divergent partition writes did not merge"; return 1; } + wait_for_digest_match persistent 1 2 10 || { + log_error "persistent digests did not converge after divergent writes"; return 1; } +} + +# Writes that land while a periodic round is in flight must neither be lost nor +# counted twice. +test_push_periodic_overlap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 4 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + local before i + before=$(get_n_instances 1) + for i in $(seq 1 6); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((100 + i)))" "overlap_app_$i" "overlap_instance_$i")" \ + >/dev/null || { log_error "overlap write $i was rejected"; return 1; } + sleep 0.2 + done + + wait_for_instances 2 $((before + 6)) 15 || { + log_error "writes racing periodic sync did not arrive"; return 1; } + [ "$(get_n_instances 1)" -eq $((before + 6)) ] || { + log_error "overlapping push and sync produced duplicate instances"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# A node whose bootnode is not up yet must keep retrying and still converge. +test_delayed_bootnode_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" delayed_app delayed_instance)" \ + >/dev/null || return 1 + + sleep 2 + start_node 1 || return 1 + + local _ + for _ in $(seq 1 25); do + has_peer_addr 1 2 && has_peer_addr 2 1 && break + sleep 1 + done + { has_peer_addr 1 2 && has_peer_addr 2 1; } || { + log_error "bootnode retry did not form the cluster"; return 1; } + wait_for_instances 1 1 15 || { + log_error "data did not converge after delayed bootnode recovery"; return 1; } +} + +# Killing a node mid-sync must not wedge it; the next round has to finish the job. +test_interrupted_sync_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + local i + for i in $(seq 1 20); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((200 + i)))" "interrupt_app_$i" "interrupt_instance_$i")" \ + >/dev/null || { log_error "interrupted-sync fixture write $i was rejected"; return 1; } + done + + # Bring it up, let a sync begin, then cut it off again. The exact instant + # does not matter -- what is asserted is that the following round still + # converges, not that the cut landed at a particular byte. + start_node 2 || return 1 + setup_peers 1 2 + sleep 0.2 + stop_node 2 + + start_node 2 || return 1 + setup_peers 1 2 + wait_for_instances 2 20 20 || { + log_error "sync did not recover after interruption"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# The ephemeral store is not journalled, so after a restart it has to be rebuilt +# from peers rather than read back from disk. +test_ephemeral_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 8 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + stop_node 2 + sleep 2 + start_node 2 || return 1 + setup_peers 1 2 + + local keys1 keys2 _ + for _ in $(seq 1 20); do + keys1=$(debug_get_sync_data 1 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + keys2=$(debug_get_sync_data 2 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } && break + sleep 1 + done + + # Assert what the loop above only waited for. Matching digests are not + # enough on their own: two nodes that never found each other both hold an + # empty ephemeral store, and empty matches empty. Stubbing out setup_peers + # left this test green, which is how the gap was found. + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } || { + log_error "ephemeral stores are empty, so a digest match proves nothing: keys1=${keys1:-0} keys2=${keys2:-0}" + return 1; } + + wait_for_digest_match ephemeral 1 2 15 || { + log_error "ephemeral store did not converge after restart"; return 1; } +} + +# A joining node must be able to bootstrap from the one peer that is up, even +# though another cluster member is unreachable. +test_partial_cluster_bootstrap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 6 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" partial_app partial_instance)" \ + >/dev/null || { log_error "node 1 rejected the registration"; return 1; } + # 15s, not 10: every comparable wait in this suite allows 15, and this one + # is a peer sync over a link that setup_peers has only just established. + wait_for_instances 2 1 15 || { + log_error "node 2 never saw the instance registered on node 1"; return 1; } + + stop_node 2 + start_node 3 || return 1 + wait_for_instances 3 1 20 || { + log_error "node 3 did not bootstrap while node 2 was unavailable"; return 1; } +} + +# A node that lost its identity but kept its node id must be rejected on the +# first exchange, must not cost the established peer any data, and must then be +# allowed to rejoin under its new identity rather than staying wedged. +test_node_id_reuse_rejected() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 10 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; return 1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 400)" reuse_app reuse_fixture)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || { log_error "node 2 did not receive the recovery fixture"; return 1; } + + local old_uuid new_uuid keys_before keys_after + old_uuid=$(get_node_uuid 2) + keys_before=$(get_n_keys 1) + + + stop_node 2 + wipe_data 2 || return 1 + start_node 2 || return 1 + + new_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ -n "$new_uuid" ] && [ "$old_uuid" != "$new_uuid" ]; } || { + log_error "fresh node 2 did not receive a new UUID"; return 1; } + + setup_peers 1 2 + + local seen=false log1 log2 _ + for _ in $(seq 1 15); do + log1=$(dump_log 1); log2=$(dump_log 2) + if grep -q "UUID mismatch" "$log1" "$log2" 2>/dev/null; then seen=true; break; fi + sleep 1 + done + [ "$seen" = true ] || { log_error "reused node ID was not rejected"; return 1; } + + # Deliberate deviation from the suite this was ported from, which compared + # total key counts before and after and required the count not to drop. + # + # It does drop, by exactly one, and permanently: node 1 discards node 2's + # superseded identity record, which is the correct thing to do with it. + # Measured 9 -> 8, still 8 thirty seconds later, with the registered + # instance present throughout. So the old assertion forbids a legitimate + # deletion, and passed only when the replacement record happened to land + # before the count was taken. + # + # What the test is actually about -- its own comment says "node 1's data is + # still intact" -- is the replicated instance records, so assert on those. + local instances_after + instances_after=$(get_n_instances 1) + [ "$instances_after" -ge 1 ] 2>/dev/null || { + log_error "node 1 lost its registered instances after node 2 restarted with a reused ID: $instances_after" + return 1; } + log_info "node 1 keys ${keys_before} -> $(get_n_keys 1), instances ${instances_after}" + + wait_for_instances 2 1 20 || { + log_error "fresh node did not recover after the UUID rejection"; return 1; } + # 40s, not the 15 this asked for before. + # + # Recovery here needs the rejected node's fresh identity record to reach + # node 1 in a sync *response* -- its own requests still fail node 1's + # inbound check -- and then an anti-entropy round to carry the store, so it + # costs several 5s intervals rather than one. Measured three times on an + # idle machine: 16s, 17s, 18s. + # + # It passed at 15 only because the wait helpers used to count iterations + # instead of seconds and so ran roughly twice as long as they claimed (see + # `wait_until` in rpc.sh). With the timer made honest this became a coin + # flip -- it passed on one full run and failed on the next. The number now + # says what the operation needs, with headroom for a loaded runner. + wait_for_digest_match persistent 1 2 40 || { + log_error "stores did not converge after UUID recovery"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 2 "$(test_public_key 401)" reuse_app post_recovery)" \ + >/dev/null || return 1 + wait_for_instances 1 2 15 || { log_error "post-recovery write did not propagate"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# --------------------------------------------------------------- advanced + +# A registration must survive a restart, and the store must hold more than the +# handful of keys a bare node writes for itself. +test_client_registration_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local client_ip keys_before keys_after + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 502)" persist_app persist_inst)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + keys_before=$(get_n_keys 1) + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + + { [ "$keys_after" -ge "$keys_before" ] && [ "$keys_before" -gt 2 ]; } || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +test_stress_writes() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key app inst success=0 + for i in $(seq 1 10); do + key=$(test_public_key $((600 + i))) + app=$(printf "stressapp%02d" "$i") + inst=$(printf "stressinst%02d" "$i") + verify_register_response "$(debug_register_cvm 1 "$key" "$app" "$inst")" >/dev/null 2>&1 \ + && success=$((success + 1)) + done + sleep 2 + + local keys_after + keys_after=$(get_n_keys 1) + { [ "$success" -eq 10 ] && [ "$keys_after" -gt 2 ]; } || { + log_error "success=$success keys_after=$keys_after"; return 1; } +} + +# Writes accepted while the peer was gone must reach it afterwards, and each +# node's two views must still agree once they do. +test_network_partition() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + stop_node 2 + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((700 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "partition_app$i" "partition_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + + local kv1_during ps1_during + kv1_during=$(get_n_instances 1) + ps1_during=$(get_n_proxy_state_instances 1) + + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 15 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + # Peering is not the thing being waited for here. What the assertions below + # test is that the rejoining node caught up on everything written while it + # was away, which takes an anti-entropy round after the peer records land -- + # the blind settle this replaced covered both, and waiting only for peering + # would return before the catch-up and leave the counts to a race. + wait_for_instances 2 "$kv1_during" 15 || { + log_error "node 2 did not catch up after the partition healed"; return 1; } + + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$success" -eq 3 ] && [ "$kv1_during" -ge 3 ]; } || { + log_error "registration or KvStore write failed during partition"; ok=1; } + [ "$kv2" -ge "$kv1_during" ] || { + log_error "node 2 KvStore sync failed: kv2=$kv2 expected >= $kv1_during"; ok=1; } + [ "$ps2" -ge "$kv1_during" ] || { + log_error "node 2 ProxyState sync failed: ps2=$ps2 expected >= $kv1_during"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + log_info "ps1_during=$ps1_during" + return $ok +} + +# The same condition as `_three_node_views_agree`, without the logging. +# +# Polling the loud one would print a diagnosis on every failed attempt, so the +# log fills with the states the cluster passed through on its way to the right +# one. Wait on this, then assert with the loud one, which reports the state that +# actually stood. +_three_node_views_converged() { + local n kv ps + for n in 1 2 3; do + kv=$(get_n_instances "$n") + ps=$(get_n_proxy_state_instances "$n") + # Every condition the loud twin asserts, including the equality. Waiting + # on a weaker predicate than the one being asserted is how a wait exits + # early on a state the assertion then rejects -- `kv=2, ps=1` is a real + # intermediate here, because the two are updated by different paths. + [ "${kv:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${ps:-0}" -ge 1 ] 2>/dev/null || return 1 + [ "${kv:-0}" -eq "${ps:-1}" ] 2>/dev/null || return 1 + done + return 0 +} + +_three_node_views_agree() { + local kv1 kv2 kv3 ps1 ps2 ps3 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2); kv3=$(get_n_instances 3) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + ps3=$(get_n_proxy_state_instances 3) + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ] && [ "$kv3" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2 kv3=$kv3"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ] && [ "$ps3" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2 ps3=$ps3"; ok=1; } + { [ "$kv1" -eq "$ps1" ] && [ "$kv2" -eq "$ps2" ] && [ "$kv3" -eq "$ps3" ]; } || { + log_error "inconsistency between KvStore and ProxyState"; ok=1; } + return $ok +} + +test_three_node_cluster() { + cleanup_cluster + generate_config 1 ""; generate_config 2 ""; generate_config 3 "" + start_node 1 || return 1 + start_node 2 || return 1 + start_node 3 || return 1 + setup_peers 1 2 3 + wait_for_peers 10 1 2 3 || { + log_error "nodes 1 2 3 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 503)" threenode_app threenode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + wait_until 20 _three_node_views_converged || true + _three_node_views_agree +} + +# The same shape, but the two joiners discover the cluster through a bootnode +# instead of being told about each other. +test_three_node_bootnode() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + sleep 2 + start_node 2 || return 1 + start_node 3 || return 1 + # Discovery through the bootnode is what this test is about, so wait on it + # rather than past it. Same 15s ceiling the blind settle had. + wait_for_peers 15 1 2 3 || { + log_error "the joiners did not discover the cluster through the bootnode"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 504)" bootnode_app bootnode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + wait_until 20 _three_node_views_converged || true + _three_node_views_agree +} + +# The periodic persist must actually run -- and be seen to run -- and what it +# wrote must come back after a restart. +test_periodic_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((900 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "persist_app$i" "persist_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 3 ] || { log_error "registered only $success/3 clients"; return 1; } + + local keys_before keys_after log wal_size + keys_before=$(get_n_keys 1) + # `persist_interval` is 5s, so the line is due in one round -- but a blind + # wait of a round and a half is both slower than it needs to be and, on a + # loaded machine, shorter than it needs to be. Poll for it. + wait_until 20 _node1_logged_a_periodic_persist || true + + log=$(dump_log 1) + grep -q "periodic persist completed" "$log" || { + log_error "periodic persist message not found in the log"; return 1; } + + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null || { log_error "WAL file missing or empty"; return 1; } + log_info "WAL size after periodic persist: ${wal_size} bytes" + + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + [ "$keys_after" -ge "$keys_before" ] || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +# ------------------------------------------------------------------ admin + +test_admin_set_node_url() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local new_url="https://new-node2.example.com:8011" response stored + response=$(admin_set_node_url 1 2 "$new_url") + echo "$response" | grep -q '"error"' && { log_error "SetNodeUrl returned: $response"; return 1; } + + sleep 2 + stored=$(get_peer_url_from_sync 1 2) + [ "$stored" = "$new_url" ] || { + log_error "expected '$new_url', got '$stored'"; return 1; } +} + +test_admin_set_node_status() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + admin_set_node_url 1 2 "https://node2.example.com:8011" >/dev/null + sleep 1 + + local response + for state in down up; do + response=$(admin_set_node_status 1 2 "$state") + # Absence of "error" is not enough on its own: a request that never + # reached the node yields an empty string, which contains no "error" + # either, so a total connectivity failure would pass. + [ -n "$response" ] || { + log_error "SetNodeStatus($state) returned nothing at all"; return 1; } + echo "$response" | grep -q '"error"' && { + log_error "SetNodeStatus($state) returned: $response"; return 1; } + sleep 1 + done + + # Kept as a warning, exactly as the original had it: whether an unknown + # status is refused is not what this test is here to pin down. + response=$(admin_set_node_status 1 2 invalid) + echo "$response" | grep -q '"error"' || \ + log_warn "invalid status was not rejected (may be acceptable)" + return 0 +} + +# A node marked down must not be handed to registering CVMs, and must be handed +# out again once it is back up. +test_node_status_register_exclude() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + wait_for_peers 5 1 2 || { + log_error "nodes 1 2 did not learn about each other"; return 1; } + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + admin_set_node_status 1 2 down >/dev/null + sleep 2 + local response + response=$(debug_register_cvm 1 "$(test_public_key 505)" downtest_app downtest_inst) + verify_register_response "$response" >/dev/null || { log_error "registration failed"; return 1; } + response_lists_gateway "$response" 2 && { + log_error "node 2 (down) was included in the registration response"; return 1; } + + admin_set_node_status 1 2 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 506)" uptest_app uptest_inst2) + response_lists_gateway "$response" 2 || { + log_error "node 2 (up) was not included in the registration response"; return 1; } +} + +# A node marked down must refuse to register CVMs itself. +test_node_status_register_reject() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 507)" upnode_app upnode_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed while the node was up"; return 1; } + + admin_set_node_status 1 1 down >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 508)" downnode_app downnode_inst) + echo "$response" | grep -qi "error" || { + log_error "registration was not rejected while the node is down"; return 1; } + + admin_set_node_status 1 1 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 509)" backup_app backup_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed once the node was back up"; return 1; } +} diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator b/dstack/gateway/test-run/e2e/Dockerfile.simulator deleted file mode 100644 index 643e83c42..000000000 --- a/dstack/gateway/test-run/e2e/Dockerfile.simulator +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: © 2026 Phala Network -# SPDX-License-Identifier: Apache-2.0 - -FROM rust:1.92-bookworm AS builder -WORKDIR /src -COPY . . -RUN cargo build --manifest-path dstack/Cargo.toml --locked --release \ - -p dstack-guest-agent-simulator - -FROM debian:bookworm-slim -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ - rm -rf /var/lib/apt/lists/* -WORKDIR /opt/dstack-simulator -COPY --from=builder /src/dstack/target/release/dstack-simulator /usr/local/bin/dstack-simulator -COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \ - sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./ -COPY dstack/gateway/test-run/e2e/configs/simulator.toml ./simulator.toml -CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"] diff --git a/dstack/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml index b90efd4db..dec4e23ce 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-1.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-1.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml index c7bdd729f..975e0ab2e 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-2.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-2.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml index 0cb845126..a29b1de90 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-3.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-3.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/simulator.toml b/dstack/gateway/test-run/e2e/configs/simulator.toml deleted file mode 100644 index e7020dbca..000000000 --- a/dstack/gateway/test-run/e2e/configs/simulator.toml +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: © 2026 Phala Network -# SPDX-License-Identifier: Apache-2.0 - -[default] -workers = 8 -max_blocking = 64 -ident = "dstack Gateway E2E Simulator" -temp_dir = "/tmp" -keep_alive = 10 -log_level = "info" - -[default.core] -keys_file = "/opt/dstack-simulator/appkeys.json" -compose_file = "/opt/dstack-simulator/app-compose.json" -sys_config_file = "/opt/dstack-simulator/sys-config.json" -data_disks = ["/"] - -[default.core.simulator] -attestation_file = "/opt/dstack-simulator/attestation.bin" -patch_report_data = true - -[internal] -address = "unix:/var/run/dstack/dstack.sock" -reuse = true diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml index 7d90729ef..ae601198a 100644 --- a/dstack/gateway/test-run/e2e/docker-compose.yml +++ b/dstack/gateway/test-run/e2e/docker-compose.yml @@ -6,7 +6,14 @@ # Uses mock services: Pebble (ACME) + mock-cf-dns-api (Cloudflare DNS) # Uses a test-local dstack Guest Agent simulator for certificate and app identity flows. +# Same long-lived fixture project the cluster suite uses; run-e2e.sh brings it +# up. Referencing it rather than including it keeps one instance of the seed and +# the roots derived from it, and keeps this suite's network to itself. networks: + attestation: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set} + certbot-test: driver: bridge ipam: @@ -14,23 +21,14 @@ networks: - subnet: 172.30.0.0/24 volumes: - pebble-certs: dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + attestation-roots: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-roots services: - dstack-simulator: - build: - context: ../../../.. - dockerfile: dstack/gateway/test-run/e2e/Dockerfile.simulator - image: dstack-simulator:gateway-e2e - volumes: - - dstack-socket:/var/run/dstack - healthcheck: - test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] - interval: 1s - timeout: 1s - retries: 30 - # ==================== Mock Services ==================== # Mock Cloudflare DNS API. Built from the repo rather than pulled, and the @@ -100,6 +98,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-1 networks: + attestation: certbot-test: ipv4_address: 172.30.0.21 ports: @@ -110,14 +109,15 @@ services: volumes: - ./configs/gateway-1.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + # The fixture is a separate project now; run-e2e.sh brings it up and waits + # for it before this one starts, so it cannot be a depends_on here. depends_on: - dstack-simulator: - condition: service_healthy mock-cf-dns-api: condition: service_healthy pebble: @@ -145,6 +145,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-2 networks: + attestation: certbot-test: ipv4_address: 172.30.0.22 ports: @@ -155,6 +156,7 @@ services: volumes: - ./configs/gateway-2.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: @@ -183,6 +185,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-3 networks: + attestation: certbot-test: ipv4_address: 172.30.0.23 ports: @@ -193,6 +196,7 @@ services: volumes: - ./configs/gateway-3.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: diff --git a/dstack/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh index 7a7e0cccd..35fc0eb5e 100755 --- a/dstack/gateway/test-run/e2e/run-e2e.sh +++ b/dstack/gateway/test-run/e2e/run-e2e.sh @@ -10,6 +10,28 @@ set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +TEST_RUN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# The attestation fixture is its own long-lived project: one simulator, one +# collateral service, one seed. Both suites reference its network and volumes +# rather than each building a copy, so the seed that signs quotes and the seed +# that derives the verifying roots cannot drift apart. +# +# Defined here, above the EXIT trap that calls it -- a cleanup handler naming a +# function defined later fails on exactly the error paths it exists for. +# The attestation fixture, namespaced to this suite. +# +# Each suite runs its own copy rather than sharing one project. The seed that +# signs quotes and the seed that derives the verifying roots come from files in +# `attestation/`, so separate instances agree by construction -- while a shared +# instance meant whichever suite finished first tore it down under the others, +# and meant the three CI workflows could not run at the same time. +FIXTURE_NS="dstack-fixture-e2e" +export FIXTURE_NS + +fixture_compose() { + docker compose -p "$FIXTURE_NS" -f "$TEST_RUN_DIR/attestation/fixture.yml" "$@" +} # Colors for output RED='\033[0;31m' @@ -20,6 +42,7 @@ NC='\033[0m' # No Color log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[OK]${NC} $1"; } +# shellcheck disable=SC2317 # called from the EXIT trap, which shellcheck cannot see log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } @@ -46,6 +69,10 @@ while [[ $# -gt 0 ]]; do cd "$SCRIPT_DIR" log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true + # The fixture too: it is a project of its own, so a suite-only + # teardown leaves it and its global network and volumes behind on + # any runner that outlives the job. + fixture_compose down -v --remove-orphans 2>/dev/null || true log_success "Containers stopped" exit 0 ;; @@ -72,10 +99,39 @@ done cd "$SCRIPT_DIR" # Cleanup function +# Two runs share one compose project, one set of container names and one work +# directory, so a second run tears down what the first is using. The damage does +# not look like a collision -- it looks like flaky tests, because state vanishes +# mid-run. Refuse to start instead. +# Locked on the directory, not on a file inside it. Deleting a lock FILE does +# not release the lock, it detaches it: the holder keeps its inode while the +# next run creates a fresh one and flocks that successfully. Three runs of the +# cluster suite overlapped exactly that way -- each wiping the others' state, +# with the failures reading as flaky sync rather than as a collision -- because +# a "clean up before rerunning" step had removed the lock file by name. +exec 9<"$SCRIPT_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($SCRIPT_DIR)" + exit 1 +fi + +# Probed BEFORE the trap is installed, not next to the `up` that follows it: any +# failure in between would otherwise run cleanup with the variable unset, take +# the `:-0` default, and tear down a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +# shellcheck disable=SC2317 # the body runs from the EXIT trap installed below cleanup() { if ! $KEEP_RUNNING; then log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the + # next run fail with "network dstack-attestation declared as external, + # but could not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi fi } @@ -107,28 +163,16 @@ if ! $SKIP_BUILD; then fi # Step 2: Create gateway docker image (alpine for musl) +# +# Through the shared builder, not a private copy of the Dockerfile. This used to +# heredoc its own byte-identical `Dockerfile.gateway`, build from it and `rm` it +# afterwards -- which defeated the point of `build-gateway-image.sh` ("shared so +# the two suites cannot end up testing different builds"), and, because the `rm` +# was neither conditional nor trapped, left an untracked `e2e/Dockerfile.gateway` +# in the tree whenever the build failed under `set -e`. log_info "Creating gateway docker image..." cd "$SCRIPT_DIR" - -cat > Dockerfile.gateway << 'EOF' -FROM alpine:latest - -RUN apk add --no-cache \ - wireguard-tools \ - iproute2 \ - curl \ - ca-certificates - -COPY dstack-gateway /usr/local/bin/dstack-gateway - -RUN chmod +x /usr/local/bin/dstack-gateway && \ - mkdir -p /etc/gateway/certs /var/lib/gateway - -ENTRYPOINT ["/usr/local/bin/dstack-gateway", "-c", "/etc/gateway/gateway.toml"] -EOF - -docker build -t dstack-gateway:test -f Dockerfile.gateway . -rm Dockerfile.gateway +"$TEST_RUN_DIR/build-gateway-image.sh" "$SCRIPT_DIR" --skip-build log_success "Gateway image created: dstack-gateway:test" # Step 3: Run docker compose @@ -141,8 +185,18 @@ export GATEWAY_IMAGE=dstack-gateway:test # shared with the full-stack suite, so a stale copy from an older checkout # would be picked up silently -- and a mock that does not answer TCP fails # every challenge now that Pebble actually validates them. -log_info "Building the mock DNS/Cloudflare API..." -docker compose build mock-cf-dns-api +# Build every image rather than trusting whatever carries its tag. `image:` plus +# `build:` means compose reuses a local tag if one exists, and these tags are +# shared with other suites and other checkouts, so a stale copy would be picked +# up silently -- a mock that does not answer TCP fails every challenge now that +# Pebble validates them, and a stale simulator issues certificates from an older +# cert-client, which is the code path the cluster's app_id pinning depends on. +log_info "Building the mock services and the guest agent simulator..." +docker compose build +fixture_compose build + +log_info "Starting the attestation fixture..." +fixture_compose up -d --wait docker compose up -d mock-cf-dns-api pebble log_info "Waiting for mock services to be healthy..." @@ -154,8 +208,11 @@ sleep 10 # Step 4: Run tests log_info "Running tests..." -docker compose run --rm test-runner -TEST_EXIT_CODE=$? +# `set -e` is on, so a plain call would abort the script the moment the suite +# fails and the report below would be unreachable -- the failure still exits +# non-zero, but silently. Capture the status instead of inheriting it. +TEST_EXIT_CODE=0 +docker compose run --rm test-runner || TEST_EXIT_CODE=$? # Step 5: Report result (cleanup handled by trap) if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh index 9eecb98f7..66c224431 100755 --- a/dstack/gateway/test-run/e2e/test.sh +++ b/dstack/gateway/test-run/e2e/test.sh @@ -629,12 +629,24 @@ setup_certbot_config() { -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \ || log_warn "AddZtDomain failed for $domain (may already exist)" + # The first domain on a fresh deployment has to create the global ACME + # account, and it holds the shared ACME lock while it does. A domain + # added inside that window is refused with "retry after it finishes" + # and nothing retries it promptly, so adding three in a tight loop + # leaves two without certificates. Retry here, as the error asks. log_info "Triggering renewal for: $domain" - curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ - -H "${ADMIN_AUTH_HEADER}" \ - -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null || \ - log_warn "Renewal request failed for $domain (may retry)" + local attempt=1 + while [ $attempt -le 10 ]; do + if curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ + -H "${ADMIN_AUTH_HEADER}" \ + -H "Content-Type: application/json" \ + -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null; then + break + fi + attempt=$((attempt + 1)) + sleep 3 + done + [ $attempt -le 10 ] || log_warn "Renewal never succeeded for $domain after 10 tries" done return 0 @@ -701,26 +713,26 @@ main() { # Phase 5: Certificate issuance log_phase 5 "Certificate issuance" - local first_domain first_sni first_proxy - first_domain=$(echo "$CERT_DOMAINS" | cut -d' ' -f1) - first_sni=$(get_test_sni "$first_domain") + local first_proxy first_proxy=$(echo "$GATEWAY_PROXIES" | cut -d' ' -f1) - log_info "Waiting for certificates (up to 120s)..." - local waited=0 - while [ $waited -lt 120 ]; do - if test_certificate_issued "$first_proxy" "$first_sni"; then - log_info "Certificate detected for $first_sni" - break - fi - sleep 5 - waited=$((waited + 5)) - log_info "Waiting... (${waited}s)" - done - - local sni wildcard + # Wait for each domain, not just the first. The orders run concurrently and + # finish in any order, so the first one landing says nothing about the rest + # -- asserting on all three the moment it does made the last domain a + # coin flip. + local sni waited for domain in $CERT_DOMAINS; do sni=$(get_test_sni "$domain") + log_info "Waiting for certificate: $domain (up to 120s)" + waited=0 + while [ $waited -lt 120 ]; do + if test_certificate_issued "$first_proxy" "$sni"; then + log_info "Certificate detected for $sni" + break + fi + sleep 5 + waited=$((waited + 5)) + done run_test "Certificate issued for $domain" \ "$(test_certificate_issued "$first_proxy" "$sni"; echo $?)" done diff --git a/dstack/gateway/test-run/proxy-e2e/Dockerfile b/dstack/gateway/test-run/proxy-e2e/Dockerfile new file mode 100644 index 000000000..f6852017b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/Dockerfile @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# The proxy data-path suite runs the gateway, the origin server and the probe +# client together in one container, because they must share a network +# namespace: `insecure_localhost_backend` resolves the backend to 127.0.0.1 as +# seen by the gateway itself. +FROM debian:bookworm-slim + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends \ + python3 openssl iproute2 iputils-ping curl ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /suite +COPY dstack-gateway /usr/local/bin/dstack-gateway +RUN chmod +x /usr/local/bin/dstack-gateway +COPY test_proxy.sh /suite/test_proxy.sh +COPY proxy /suite/proxy +RUN chmod +x /suite/test_proxy.sh + +ENV GATEWAY_BIN=/usr/local/bin/dstack-gateway +ENTRYPOINT ["/suite/test_proxy.sh"] diff --git a/dstack/gateway/test-run/proxy-e2e/README.md b/dstack/gateway/test-run/proxy-e2e/README.md new file mode 100644 index 000000000..f9004880b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/README.md @@ -0,0 +1,34 @@ + + +# Gateway proxy data-path suite + +Runs the gateway, the origin server and the probe client in one container. They +have to share a network namespace: `insecure_localhost_backend` resolves the +backend address to `127.0.0.1` as the gateway itself sees it. + +## Why the kTLS fallback arm runs in a second container + +One arm asserts that a gateway configured for kTLS on a kernel without the TLS +ULP falls back to userspace instead of truncating a gated transfer at the gate. +The suite used to produce that condition with `sudo rmmod tls`, which cannot +work in a container and which took the module away from the whole host. + +`notls-seccomp.json` makes `setsockopt(IPPROTO_TCP, TCP_ULP)` return +`ENOPROTOOPT` for one container instead, which is exactly what `probe_ktls()` +sees on a kernel without `CONFIG_TLS`. It touches nothing outside that +container and does not care what else on the machine is using TLS. + +Note what the profile costs: `defaultAction` is `SCMP_ACT_ALLOW`, and a +`security_opt: seccomp=` *replaces* Docker's default profile rather than +extending it. So this arm runs with one syscall filtered and everything else +permitted, where the main arm runs under the default profile. That is acceptable +for a test container on a throwaway network namespace, and it is the price of +producing the condition at all -- but the two arms are not syscall-equivalent, +and a finding that reproduces only here should be checked against that. + +A seccomp profile is fixed when a container is created, so the arm needs its +own container rather than a restart of the main one. diff --git a/dstack/gateway/test-run/proxy-e2e/docker-compose.yml b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml new file mode 100644 index 000000000..595b07404 --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Two containers from one image. The suite restarts the gateway ~25 times with a +# different config each round; that loop stays inside a container rather than +# being handed to compose, which would turn every arm into a container create. + +# The gateway asks the guest agent for its own app id at startup. That is a unix +# socket, not a network hop, so sharing it through a volume leaves this suite's +# "everything in one network namespace" requirement untouched -- these services +# stay on network_mode: bridge and never join the fixture's network. +# +# Referenced rather than included, like the other two suites: one fixture +# project, one seed, one set of roots derived from it. +volumes: + dstack-socket: + external: true + name: ${FIXTURE_NS:?FIXTURE_NS must be set}-socket + +x-suite: &suite + build: + context: . + dockerfile: Dockerfile + image: dstack-gateway-proxy-tests:local + cap_add: + - NET_ADMIN + volumes: + - dstack-socket:/var/run/dstack + # The suite's work directory, so the per-arm gateway logs outlive the + # container and CI has something to attach when an arm fails. + - ./run:/work + environment: + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + # A subdirectory per arm. Both arms run the whole suite, so with one shared + # WORK the second overwrote every `gw-