From d5171bdf072939bc41f03f877e1ba8800f37d92f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:47:12 -0700 Subject: [PATCH 01/27] fix(cert-client): stamp the app id on locally issued certificates KMS stamps the app_id it verified into the certificate it returns. The local CA branch has the same value available -- the CSR carries the attestation it was derived from -- but dropped it, so every certificate issued through a local CA came back without the extension. A peer that pins app_id therefore rejected all of them. dstack-gateway's cluster mTLS does exactly that, which left clustering working under a KMS key provider and silently broken under every other one. The decode is best effort on purpose: an app whose attestation carries no app-id event decodes to an empty value, and stamping that would make every such peer match every other. Absent stays absent, and the peer rejects it as it did before. --- dstack/cert-client/src/lib.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/dstack/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs index e55689b52..39958b0a4 100644 --- a/dstack/cert-client/src/lib.rs +++ b/dstack/cert-client/src/lib.rs @@ -31,8 +31,28 @@ impl CertRequestClient { ) -> Result> { match self { CertRequestClient::Local { ca } => { + // KMS stamps the app_id it verified into the certificate. The + // local CA has the same value already -- the CSR carries the + // attestation it comes from -- but used to drop it, so a peer + // that pins app_id rejected every certificate issued here. + // dstack-gateway's cluster mTLS does exactly that, which left + // clustering working under a KMS key provider and silently + // broken under every other one. + // + // Best effort on purpose: an app whose attestation carries no + // app-id event decodes to an empty one, and stamping that + // would make every such peer match every other. Absent stays + // absent, and the peer rejects it as before. + let app_id = csr + .attestation + .clone() + .into_v1() + .decode_app_info(false) + .ok() + .map(|info| info.app_id) + .filter(|app_id| !app_id.is_empty()); let cert = ca - .sign_csr(csr, None, "app:custom") + .sign_csr(csr, app_id.as_deref(), "app:custom") .context("Failed to sign certificate")?; Ok(vec![cert.pem(), ca.pem_cert.clone()]) } From 9d8824827cf206c6f2ee85c78d23c5bd39bcbc48 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:47:19 -0700 Subject: [PATCH 02/27] refactor(gateway): split the wavekv sync handlers from their routes The sync routes are the cluster's write surface: anything that reaches them can insert entries that replicate to every gateway. `verify_gateway_peer` is the only thing in front of them, and no test had ever executed it -- every test reached the body below by turning the check off, and Rocket's local client speaks no TLS, so it can never present the certificate the check wants. Replacing the whole function body with `Ok(())` did not turn the suite red. `handle_sync` and `handle_push` now hold everything the routes do once the caller is known to be a peer, so the tests that are about the gzip framing, the store split, the uuid check and the removed-sender refusal call them directly, and reach that code without going near the check. That frees the check to be tested for what it is: a request through the real route, with no certificate, is refused. One test does present a certificate, over a real mutually authenticated TLS connection, so the accepting half is covered too. --- dstack/gateway/src/web_routes/wavekv_sync.rs | 309 ++++++++++++------- 1 file changed, 198 insertions(+), 111 deletions(-) diff --git a/dstack/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs index b1a00bf51..066795b9e 100644 --- a/dstack/gateway/src/web_routes/wavekv_sync.rs +++ b/dstack/gateway/src/web_routes/wavekv_sync.rs @@ -72,8 +72,14 @@ async fn read_compressed_body(data: Data<'_>) -> Result, Status> { } /// Read a sync envelope from a bounded request body. -async fn read_envelope(data: Data<'_>) -> Result { - let decompressed = gunzip(&read_compressed_body(data).await?)?; +/// Decode a gzipped sync envelope. +/// +/// Split from the `Data` read so the request-handling half of these endpoints +/// can be driven from a test. Rocket's local client presents no certificate, so +/// a test that went through the real route would be stopped at the peer check +/// before reaching any of this. +fn decode_envelope(compressed: &[u8]) -> Result { + let decompressed = gunzip(compressed)?; // `SyncEnvelope::decode` enforces the schema version and rejects trailing bytes; // it is deliberately not the generic `decode` used for KV values. SyncEnvelope::decode(&decompressed).map_err(|e| { @@ -139,12 +145,21 @@ pub async fn sync_store( data: Data<'_>, ) -> Result<(ContentType, Vec), Status> { verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) +} +/// Everything `sync_store` does once the caller is known to be a peer. +pub(crate) fn handle_sync( + state: &Proxy, + store: &str, + body: &[u8], +) -> Result<(ContentType, Vec), Status> { let Some(ref wavekv_sync) = state.wavekv_sync else { return Err(Status::ServiceUnavailable); }; - let env = read_envelope(data).await?; + let env = decode_envelope(body)?; if env.sender_id == 0 { warn!("rejected sync from invalid node_id 0"); return Err(Status::BadRequest); @@ -199,12 +214,17 @@ pub async fn push_store( data: Data<'_>, ) -> Result { verify_gateway_peer(state, cert)?; + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) +} +/// Everything `push_store` does once the caller is known to be a peer. +pub(crate) fn handle_push(state: &Proxy, store: &str, body: &[u8]) -> Result { let Some(ref wavekv_sync) = state.wavekv_sync else { return Err(Status::ServiceUnavailable); }; - let env = read_envelope(data).await?; + let env = decode_envelope(body)?; if env.sender_id == 0 { warn!("rejected push from invalid node_id 0"); return Err(Status::BadRequest); @@ -233,6 +253,9 @@ mod tests { const ME: u32 = 1; const PEER: u32 = 2; + /// Both sides of the local mTLS test are this gateway, so one id serves for + /// the certificate it presents and the identity it expects of a peer. + const TEST_APP_ID: &[u8] = b"test-app-id-0000-0001"; fn peer_uuid() -> Vec { b"the-real-peer-2".to_vec() @@ -250,8 +273,24 @@ mod tests { let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); let leaf_key = KeyPair::generate().expect("leaf key"); - let leaf_params = + let mut leaf_params = CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); + // No test here turns the peer check off, so a certificate without an app_id + // is refused before any of them reach what they are about. Stamping one + // here is also what a real peer's certificate carries. + // A DER OCTET STRING, which is what `ra_tls` writes into this extension. + // Hand-encoded rather than pulling in an ASN.1 crate for three bytes of + // header; the short-form length is valid because the payload is under 128 + // bytes, which a debug_assert pins. + debug_assert!(TEST_APP_ID.len() < 128); + let mut app_id_der = vec![0x04, TEST_APP_ID.len() as u8]; + app_id_der.extend_from_slice(TEST_APP_ID); + leaf_params + .custom_extensions + .push(ra_tls::rcgen::CustomExtension::from_oid_content( + ra_tls::oids::PHALA_RATLS_APP_ID, + app_id_der, + )); let leaf_cert = leaf_params .signed_by(&leaf_key, &ca_cert, &ca_key) .expect("leaf cert"); @@ -272,27 +311,41 @@ mod tests { } } - /// A gateway serving the real sync routes over Rocket's local client. + /// A gateway whose request-handling half these tests drive directly. /// - /// `insecure_skip_attestation` is on, which makes `verify_gateway_peer` return - /// immediately: these tests are about everything below it — route dispatch, the gzip - /// framing, the store split, the uuid check. `enforcing_gateway` covers the gate - /// itself, which this fixture cannot, because Rocket's local client speaks no TLS - /// and so can never present a certificate. - async fn serving_gateway(sync_enabled: bool) -> (Client, Proxy, TempDir) { - serving_gateway_with(sync_enabled, true).await + /// They are about everything below the peer check — the gzip framing, the store + /// split, the uuid check, the removed-sender refusal — and they call `handle_sync` + /// and `handle_push` rather than going through the routes, because the peer check runs + /// on every request these tests make and Rocket's local client speaks no TLS, so a + /// request through the route can never get past it. `enforcing_gateway` covers the check itself. + async fn serving_gateway(sync_enabled: bool) -> (Proxy, TempDir) { + let (_client, proxy, tmp) = serving_gateway_with(sync_enabled).await; + (proxy, tmp) + } + + /// The same gateway, reached through the real routes, where every request is refused + /// for want of a certificate. That refusal is the assertion. + async fn enforcing_gateway() -> (Client, Proxy, TempDir) { + serving_gateway_with(true).await } - /// The same gateway with the attestation bypass switched off, so the peer check runs - /// for real. - async fn enforcing_gateway() -> (Client, Proxy, TempDir) { - serving_gateway_with(true, false).await + /// Drive the sync endpoint's body the way its route would. + fn post_sync(proxy: &Proxy, store: &str, body: Vec) -> (Status, Vec) { + match handle_sync(proxy, store, &body) { + Ok((_content_type, bytes)) => (Status::Ok, bytes), + Err(status) => (status, Vec::new()), + } } - async fn serving_gateway_with( - sync_enabled: bool, - skip_attestation: bool, - ) -> (Client, Proxy, TempDir) { + /// Drive the push endpoint's body the way its route would. + fn post_push(proxy: &Proxy, store: &str, body: Vec) -> Status { + match handle_push(proxy, store, &body) { + Ok(status) => status, + Err(status) => status, + } + } + + async fn serving_gateway_with(sync_enabled: bool) -> (Client, Proxy, TempDir) { // `main` installs this once at startup; the sync client builds a rustls config, // so a test that skips it panics inside rustls rather than failing an assertion. let _ = rustls::crypto::ring::default_provider().install_default(); @@ -310,12 +363,11 @@ mod tests { .join("wg.conf") .to_string_lossy() .into_owned(); - config.debug.insecure_skip_attestation = skip_attestation; let tls_config = write_tls_material(temp_dir.path()); let proxy = Proxy::new(ProxyOptions { config, - my_app_id: None, + my_app_id: Some(TEST_APP_ID.to_vec()), tls_config, }) .await @@ -330,10 +382,9 @@ mod tests { /// The sync routes are the cluster's write surface: anything that reaches them can /// insert entries that replicate to every gateway. `verify_gateway_peer` is the only - /// thing standing in front of them, and with `insecure_skip_attestation` set — which - /// every other test here sets — its first statement returns `Ok(())`, so the gate - /// itself was never executed by any test. Replacing the whole function body with - /// `Ok(())` did not turn the suite red. + /// thing standing in front of them, and no test had ever executed it: every test + /// here reached the body below by turning the check off. Replacing the whole + /// function body with `Ok(())` did not turn the suite red. /// /// Rocket's local client speaks no TLS and so presents no certificate, which is /// exactly the case that must be refused. @@ -395,7 +446,7 @@ mod tests { /// The rule the sync routes are defended by: same app id or nothing. /// /// Every case below was previously unreachable, because the only tests that touched - /// this code set `insecure_skip_attestation` and returned before it. Inverting the + /// this code returned before it without checking anything. Inverting the /// comparison to `==` left the suite green. #[test] fn a_peer_is_authorized_only_when_its_app_id_matches_ours() { @@ -488,16 +539,16 @@ mod tests { #[tokio::test] async fn a_stamped_push_is_accepted_and_lands_in_the_store() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/push/persistent") - .body(body(&push_envelope(peer_uuid(), "node/9"))) - .dispatch() - .await; + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(peer_uuid(), "node/9")), + ); - assert_eq!(response.status(), Status::Ok); + assert_eq!(status, Status::Ok); assert!( proxy.kv_store().persistent().read().get("node/9").is_some(), "a well-formed push must reach the store" @@ -509,16 +560,16 @@ mod tests { /// `check_uuid` — which only the manager runs, not `merge_push` — rejected it. #[tokio::test] async fn an_unstamped_push_is_refused_at_the_route() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/push/persistent") - .body(body(&push_envelope(Vec::new(), "node/9"))) - .dispatch() - .await; + let status = post_push( + &proxy, + "persistent", + body(&push_envelope(Vec::new(), "node/9")), + ); - assert_eq!(response.status(), Status::InternalServerError); + assert_eq!(status, Status::InternalServerError); assert!( proxy.kv_store().persistent().read().get("node/9").is_none(), "a push that fails the identity check must not write anything" @@ -527,7 +578,7 @@ mod tests { #[tokio::test] async fn a_sync_round_trip_returns_a_decodable_envelope() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); proxy .kv_store() @@ -537,14 +588,9 @@ mod tests { .expect("seed"); let request = SyncEnvelope::new(PEER, peer_uuid()); - let response = client - .post("/wavekv/sync/persistent") - .body(body(&request)) - .dispatch() - .await; - - assert_eq!(response.status(), Status::Ok); - let bytes = response.into_bytes().await.expect("body"); + let (status, bytes) = post_sync(&proxy, "persistent", body(&request)); + + assert_eq!(status, Status::Ok); let decoded = SyncEnvelope::decode(&gunzip(&bytes).expect("gunzip")).expect("decode"); assert_eq!(decoded.sender_id, ME); assert!( @@ -560,7 +606,7 @@ mod tests { async fn sync_and_push_cross_a_real_mutually_authenticated_tls_connection() { use rocket::{mtls::MtlsConfig, tls::TlsConfig as RocketTlsConfig}; - let (_local, proxy, tmp) = serving_gateway(true).await; + let (proxy, tmp) = serving_gateway(true).await; register_peer(&proxy); let tls = write_tls_material(tmp.path()); @@ -651,9 +697,42 @@ mod tests { server.await.expect("server task"); } + /// The production routes with the peer check removed. + /// + /// Mounted by one test, and only because the body cap it asserts lives in + /// the route half: `handle_sync` never sees a `Data`, so a test calling it + /// directly cannot reach the limit, and Rocket's local client cannot present + /// the certificate the real route now requires. Two duplicated lines are + /// cheaper than losing coverage of a bound that exists to stop a peer -- or a + /// stranger who got that far -- from making the gateway read without end. + #[post("/wavekv/sync/", data = "")] + async fn ungated_sync( + state: &State, + store: &str, + data: Data<'_>, + ) -> Result<(ContentType, Vec), Status> { + let body = read_compressed_body(data).await?; + handle_sync(state, store, &body) + } + + #[post("/wavekv/push/", data = "")] + async fn ungated_push( + state: &State, + store: &str, + data: Data<'_>, + ) -> Result { + let body = read_compressed_body(data).await?; + handle_push(state, store, &body) + } + #[tokio::test] async fn an_oversized_compressed_request_is_rejected_explicitly() { - let (client, _proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; + let rocket = rocket::build() + .manage(proxy) + .mount("/", rocket::routes![ungated_sync, ungated_push]); + let client = Client::tracked(rocket).await.expect("rocket client"); + for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { let response = client .post(path) @@ -667,16 +746,12 @@ mod tests { /// Unknown stores are rejected rather than being routed to either replicated store. #[tokio::test] async fn an_unknown_store_is_rejected() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); - let response = client - .post("/wavekv/sync/bogus") - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; + let (status, _) = post_sync(&proxy, "bogus", body(&SyncEnvelope::new(PEER, peer_uuid()))); - assert_eq!(response.status(), Status::NotFound); + assert_eq!(status, Status::NotFound); } /// The lockout at the door. The app-identity check proves the sender is @@ -686,53 +761,65 @@ mod tests { /// SetNodeUrl re-admission path -- opens the door again. #[tokio::test] async fn a_removed_nodes_envelopes_are_refused_at_the_door() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); proxy.kv_store().mark_peer_removed(PEER).expect("mark"); - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client - .post(path) - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::Forbidden, - "{path} must refuse a removed sender" - ); - } + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::Forbidden, + "sync must refuse a removed sender" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::Forbidden, + "push must refuse a removed sender" + ); proxy.kv_store().clear_peer_removed(PEER).expect("clear"); - let response = client - .post("/wavekv/sync/persistent") - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::Ok, - "re-admission opens the door again" + let (readmitted, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), ); + assert_eq!(readmitted, Status::Ok, "re-admission opens the door again"); } /// A node with synchronization disabled reports that the service is unavailable. #[tokio::test] async fn a_sync_disabled_node_answers_503() { - let (client, _proxy, _tmp) = serving_gateway(false).await; + let (proxy, _tmp) = serving_gateway(false).await; - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client - .post(path) - .body(body(&SyncEnvelope::new(PEER, peer_uuid()))) - .dispatch() - .await; - assert_eq!( - response.status(), - Status::ServiceUnavailable, - "{path} must not look like a missing v2 route" - ); - } + let (sync_status, _) = post_sync( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + sync_status, + Status::ServiceUnavailable, + "sync must not look like a missing v2 route" + ); + let push_status = post_push( + &proxy, + "persistent", + body(&SyncEnvelope::new(PEER, peer_uuid())), + ); + assert_eq!( + push_status, + Status::ServiceUnavailable, + "push must not look like a missing v2 route" + ); } /// gzip expands by three orders of magnitude on attacker-chosen input, so the @@ -741,7 +828,7 @@ mod tests { /// has to hold against a peer running a buggy build, not just against a stranger. #[tokio::test] async fn a_compression_bomb_is_refused_before_it_is_decompressed() { - let (client, _proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; // ~130 MiB of zeroes compresses to well under the request cap. let bomb = gzip(&vec![0u8; MAX_DECOMPRESSED_SYNC_BYTES + 1]).expect("gzip"); @@ -751,14 +838,18 @@ mod tests { bomb.len() ); - for path in ["/wavekv/sync/persistent", "/wavekv/push/persistent"] { - let response = client.post(path).body(bomb.clone()).dispatch().await; - assert_eq!( - response.status(), - Status::BadRequest, - "{path} must refuse an over-sized expansion" - ); - } + let (sync_status, _) = post_sync(&proxy, "persistent", bomb.clone()); + assert_eq!( + sync_status, + Status::BadRequest, + "sync must refuse an over-sized expansion" + ); + let push_status = post_push(&proxy, "persistent", bomb); + assert_eq!( + push_status, + Status::BadRequest, + "push must refuse an over-sized expansion" + ); } /// The limits must leave room for the largest legitimate message. @@ -805,17 +896,13 @@ mod tests { #[tokio::test] async fn a_push_from_node_id_zero_is_refused() { - let (client, proxy, _tmp) = serving_gateway(true).await; + let (proxy, _tmp) = serving_gateway(true).await; register_peer(&proxy); let mut env = push_envelope(peer_uuid(), "node/9"); env.sender_id = 0; - let response = client - .post("/wavekv/push/persistent") - .body(body(&env)) - .dispatch() - .await; + let status = post_push(&proxy, "persistent", body(&env)); - assert_eq!(response.status(), Status::BadRequest); + assert_eq!(status, Status::BadRequest); } } From 850fdc5875e4c1d28fed6c8e160074a2fc553f45 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:47:31 -0700 Subject: [PATCH 03/27] test(gateway): share one attestation fixture and gateway image The suites that follow all need the same two things: a gateway container built from the tree, and a guest agent whose quotes verify without TDX hardware. Both were reachable only from the e2e directory. The simulator signs its quotes under trust anchors derived from a seed, and the collateral service reconstructs the matching public roots from that same seed. The two are useless apart and useless with different seeds, so they move into one `attestation/fixture.yml` that a suite includes, rather than being copied per suite where they would drift and leave peer quotes failing to verify for no visible reason. The fixture names no network, so each suite attaches it to its own. `build-gateway-image.sh` builds the static musl binary and wraps it in the alpine runtime image, so two suites cannot end up testing different builds. --- dstack/gateway/test-run/.gitignore | 8 +++ dstack/gateway/test-run/Dockerfile.gateway | 21 ++++++ .../attestation/Dockerfile.mock-attestation | 16 +++++ .../Dockerfile.mock-attestation.dockerignore} | 0 .../{e2e => attestation}/Dockerfile.simulator | 2 +- .../Dockerfile.simulator.dockerignore | 9 +++ .../gateway/test-run/attestation/fixture.yml | 69 +++++++++++++++++++ .../configs => attestation}/simulator.toml | 6 ++ .../test-run/attestation/tee-simulator.json | 5 ++ .../attestation/tee-simulator.json.license | 3 + .../gateway/test-run/build-gateway-image.sh | 34 +++++++++ 11 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 dstack/gateway/test-run/Dockerfile.gateway create mode 100644 dstack/gateway/test-run/attestation/Dockerfile.mock-attestation rename dstack/gateway/test-run/{e2e/Dockerfile.simulator.dockerignore => attestation/Dockerfile.mock-attestation.dockerignore} (100%) rename dstack/gateway/test-run/{e2e => attestation}/Dockerfile.simulator (92%) create mode 100644 dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore create mode 100644 dstack/gateway/test-run/attestation/fixture.yml rename dstack/gateway/test-run/{e2e/configs => attestation}/simulator.toml (56%) create mode 100644 dstack/gateway/test-run/attestation/tee-simulator.json create mode 100644 dstack/gateway/test-run/attestation/tee-simulator.json.license create mode 100755 dstack/gateway/test-run/build-gateway-image.sh diff --git a/dstack/gateway/test-run/.gitignore b/dstack/gateway/test-run/.gitignore index b1c90f813..ba4042005 100644 --- a/dstack/gateway/test-run/.gitignore +++ b/dstack/gateway/test-run/.gitignore @@ -1,4 +1,12 @@ /run/ .env /e2e/dstack-gateway +/cluster/dstack-gateway +/cluster/run/ __pycache__/ +/proxy-e2e/dstack-gateway +/proxy-e2e/test_proxy.sh +/proxy-e2e/proxy/ +/proxy-e2e/run/ +*.lock +.suite.lock diff --git a/dstack/gateway/test-run/Dockerfile.gateway b/dstack/gateway/test-run/Dockerfile.gateway new file mode 100644 index 000000000..66cc90769 --- /dev/null +++ b/dstack/gateway/test-run/Dockerfile.gateway @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Runtime image for a gateway built as a static musl binary. Shared by the +# gateway test suites, which each copy their own `dstack-gateway` next to their +# compose file before building. +FROM alpine:latest + +RUN apk add --no-cache \ + wireguard-tools \ + iproute2 \ + curl \ + ca-certificates + +COPY dstack-gateway /usr/local/bin/dstack-gateway + +RUN chmod +x /usr/local/bin/dstack-gateway && \ + mkdir -p /etc/gateway/certs /var/lib/gateway + +ENTRYPOINT ["/usr/local/bin/dstack-gateway", "-c", "/etc/gateway/gateway.toml"] diff --git a/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation new file mode 100644 index 000000000..342255e01 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src +COPY . . +RUN cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p mock-attestation + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* +COPY --from=builder /src/dstack/target/release/dstack-mock-attestation /usr/local/bin/ +COPY dstack/gateway/test-run/attestation/tee-simulator.json /etc/dstack/tee-simulator.json +ENTRYPOINT ["/usr/local/bin/dstack-mock-attestation"] diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore similarity index 100% rename from dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore rename to dstack/gateway/test-run/attestation/Dockerfile.mock-attestation.dockerignore diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator b/dstack/gateway/test-run/attestation/Dockerfile.simulator similarity index 92% rename from dstack/gateway/test-run/e2e/Dockerfile.simulator rename to dstack/gateway/test-run/attestation/Dockerfile.simulator index 643e83c42..dc7ec3d2a 100644 --- a/dstack/gateway/test-run/e2e/Dockerfile.simulator +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator @@ -15,5 +15,5 @@ WORKDIR /opt/dstack-simulator COPY --from=builder /src/dstack/target/release/dstack-simulator /usr/local/bin/dstack-simulator COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \ sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./ -COPY dstack/gateway/test-run/e2e/configs/simulator.toml ./simulator.toml +COPY dstack/gateway/test-run/attestation/simulator.toml ./simulator.toml CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"] diff --git a/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore new file mode 100644 index 000000000..f53ffe346 --- /dev/null +++ b/dstack/gateway/test-run/attestation/Dockerfile.simulator.dockerignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.git +**/target +**/node_modules +**/__pycache__ +**/.env +**/.env.* diff --git a/dstack/gateway/test-run/attestation/fixture.yml b/dstack/gateway/test-run/attestation/fixture.yml new file mode 100644 index 000000000..9088645e6 --- /dev/null +++ b/dstack/gateway/test-run/attestation/fixture.yml @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Shared attestation fixture for the gateway test suites. +# +# The guest agent simulator signs its quotes under trust anchors derived from +# `mock_attestation_seed`, and the collateral service reconstructs the matching +# public roots from the same seed. The two are only useful together and only +# while the seed matches, so they live in one file that every suite includes +# rather than being copied per suite, where they would drift apart and leave +# every peer quote failing to verify for no visible reason. +# +# Deliberately network-agnostic: an including file attaches these services to +# its own network. Compose merges that in while keeping what is defined here. + +services: + dstack-simulator: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.simulator + image: dstack-simulator:gateway-e2e + volumes: + - dstack-socket:/var/run/dstack + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] + interval: 1s + timeout: 1s + retries: 30 + + mock-attestation: + networks: [attestation] + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/attestation/Dockerfile.mock-attestation + image: dstack-mock-attestation:gateway-e2e + # No container_name on purpose. A fixed one is global, so pinning it here + # would stop two suites -- which all include this file -- from being up at + # the same time. Compose still publishes `mock-attestation` as a network + # alias, so everything addressing it by name keeps working. + command: + - serve + - --listen + - 0.0.0.0:8088 + - --config + - /etc/dstack/tee-simulator.json + - --output + - /roots + volumes: + - attestation-roots:/roots + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8088/tpm/aia/root.pem"] + interval: 1s + timeout: 2s + retries: 30 + +# Named explicitly. A per-test project has to reference these from outside, and +# compose's default `_` would make that a guess about which +# project happened to create them first. +networks: + attestation: + name: dstack-attestation + +volumes: + dstack-socket: + name: dstack-attestation-socket + attestation-roots: + name: dstack-attestation-roots diff --git a/dstack/gateway/test-run/e2e/configs/simulator.toml b/dstack/gateway/test-run/attestation/simulator.toml similarity index 56% rename from dstack/gateway/test-run/e2e/configs/simulator.toml rename to dstack/gateway/test-run/attestation/simulator.toml index e7020dbca..57e61b36b 100644 --- a/dstack/gateway/test-run/e2e/configs/simulator.toml +++ b/dstack/gateway/test-run/attestation/simulator.toml @@ -18,6 +18,12 @@ data_disks = ["/"] [default.core.simulator] attestation_file = "/opt/dstack-simulator/attestation.bin" patch_report_data = true +# Re-sign the fixture quote under trust anchors derived from this seed instead +# of shipping the fixture's original, now-stale signature. The mock collateral +# service derives the matching roots from the same seed, so the gateways verify +# each other's quotes through the normal production path rather than skipping +# the check. Must stay in sync with configs/tee-simulator.json. +mock_attestation_seed = "4747474747474747474747474747474747474747474747474747474747474747" [internal] address = "unix:/var/run/dstack/dstack.sock" diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json b/dstack/gateway/test-run/attestation/tee-simulator.json new file mode 100644 index 000000000..190eceab3 --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json @@ -0,0 +1,5 @@ +{ + "platform": "dstack-tdx", + "mock_attestation_seed": "4747474747474747474747474747474747474747474747474747474747474747", + "collateral_base_url": "http://mock-attestation:8088" +} diff --git a/dstack/gateway/test-run/attestation/tee-simulator.json.license b/dstack/gateway/test-run/attestation/tee-simulator.json.license new file mode 100644 index 000000000..7ed638e2a --- /dev/null +++ b/dstack/gateway/test-run/attestation/tee-simulator.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: © 2026 Phala Network + +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/test-run/build-gateway-image.sh b/dstack/gateway/test-run/build-gateway-image.sh new file mode 100755 index 000000000..70cacaddc --- /dev/null +++ b/dstack/gateway/test-run/build-gateway-image.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Build the gateway as a static musl binary and wrap it in the alpine runtime +# image both test suites run. Shared so the two suites cannot end up testing +# different builds. +# +# Usage: build-gateway-image.sh [--skip-build] +# where to place the binary; also the docker build context + +set -e + +TEST_RUN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DSTACK="$(cd "$TEST_RUN_DIR/../.." && pwd)" + +DEST_DIR="${1:?usage: build-gateway-image.sh [--skip-build]}" +SKIP_BUILD="${2:-}" + +if [ "$SKIP_BUILD" != "--skip-build" ]; then + echo "[INFO] building dstack-gateway (musl static)..." >&2 + (cd "$REPO_DSTACK" && cargo build --release -p dstack-gateway \ + --target x86_64-unknown-linux-musl) + cp "$REPO_DSTACK/target/x86_64-unknown-linux-musl/release/dstack-gateway" "$DEST_DIR/" +fi + +if [ ! -f "$DEST_DIR/dstack-gateway" ]; then + echo "[ERROR] $DEST_DIR/dstack-gateway is missing; run without --skip-build" >&2 + exit 1 +fi + +echo "[INFO] building the gateway image..." >&2 +docker build -t dstack-gateway:test -f "$TEST_RUN_DIR/Dockerfile.gateway" "$DEST_DIR" From 900fd67b670906d8de79b2ea6ac5509cec002787 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:47:42 -0700 Subject: [PATCH 04/27] test(gateway): run the cluster suite under docker compose `test_suite.sh` ran three gateways as host processes, which is why it was never wired into CI and why it rotted: it wanted host WireGuard, host ports and a host-installed simulator, and nothing checked that any of it still worked. Its 28 tests move into `cluster/tests.sh`, driven from the host against containers. Each test gets its own compose project on a network joined to the long-lived attestation fixture, so a test that leaves a node wedged cannot reach the next one, and the fixture is built once rather than per test. `cluster.sh` goes with it, with nothing harvested. Its one distinct command registered a CVM and printed peer counts; `test_cross_node_data_sync` already registers a CVM and asserts it reaches the other node in both the KvStore and the ProxyState view, and that the two views agree. That is the same ground, asserted instead of printed. Three source comments cited the two deleted scripts for the address shapes a deployment can take. The shapes are unchanged, so they now cite the suites. --- dstack/gateway/src/config.rs | 4 +- dstack/gateway/src/kv/import.rs | 4 +- dstack/gateway/src/main_service/tests.rs | 4 +- dstack/gateway/test-run/cluster.sh | 441 --- .../test-run/cluster/docker-compose.yml | 87 + dstack/gateway/test-run/cluster/lib.sh | 280 ++ dstack/gateway/test-run/cluster/rpc.sh | 257 ++ .../test-run/cluster/run-cluster-tests.sh | 227 ++ dstack/gateway/test-run/cluster/tests.sh | 816 ++++++ dstack/gateway/test-run/test_suite.sh | 2447 ----------------- 10 files changed, 1673 insertions(+), 2894 deletions(-) delete mode 100755 dstack/gateway/test-run/cluster.sh create mode 100644 dstack/gateway/test-run/cluster/docker-compose.yml create mode 100644 dstack/gateway/test-run/cluster/lib.sh create mode 100644 dstack/gateway/test-run/cluster/rpc.sh create mode 100755 dstack/gateway/test-run/cluster/run-cluster-tests.sh create mode 100644 dstack/gateway/test-run/cluster/tests.sh delete mode 100755 dstack/gateway/test-run/test_suite.sh diff --git a/dstack/gateway/src/config.rs b/dstack/gateway/src/config.rs index fdab616ac..fa587bc28 100644 --- a/dstack/gateway/src/config.rs +++ b/dstack/gateway/src/config.rs @@ -50,8 +50,8 @@ impl WgConfig { /// `client_ip_range`. Nothing in this node's config describes the other /// nodes' pools, and the deployments do not even agree on a shape that /// could be inferred: `dstack-app/deploy-to-vmm.sh` puts every pool inside - /// one /16 that each interface covers, while `test-run/cluster.sh` and the - /// e2e configs give each node a /24 that no other node's interface covers. + /// one /16 that each interface covers, while the `test-run` suites give + /// each node a /24 that no other node's interface covers. /// Judging a replicated address by local topology refuses legitimate peers /// under the second shape, so this is limited to what a node can assert on /// its own: an ordinary unicast address that is not one of *this* gateway's. diff --git a/dstack/gateway/src/kv/import.rs b/dstack/gateway/src/kv/import.rs index 1b49d3023..593715379 100644 --- a/dstack/gateway/src/kv/import.rs +++ b/dstack/gateway/src/kv/import.rs @@ -419,8 +419,8 @@ mod tests { let shapes = [ // deploy-to-vmm.sh: /18 pools inside a shared /16 interface. ("10.8.0.1/16", "10.8.0.0/18", "10.8.0.5", "10.8.64.5"), - // test-run/cluster.sh and e2e/configs: a /24 per node, and no - // node's interface covers another's. + // The test-run suites: a /24 per node, and no node's interface + // covers another's. ("10.0.41.1/24", "10.0.41.0/24", "10.0.41.5", "10.0.42.5"), ]; for (ip, pool, mine, peers) in shapes { diff --git a/dstack/gateway/src/main_service/tests.rs b/dstack/gateway/src/main_service/tests.rs index ca5524499..8d60abcc7 100644 --- a/dstack/gateway/src/main_service/tests.rs +++ b/dstack/gateway/src/main_service/tests.rs @@ -2133,8 +2133,8 @@ async fn a_poisoned_peer_record_costs_only_its_own_instance() { #[tokio::test] async fn a_cvm_registered_on_another_node_becomes_a_wg_peer_here() { let state = create_test_state().await; - // What a peer node allocated out of its own slice. `test-run/cluster.sh` - // and the e2e configs give each node a /24 of its own, so a peer's address + // What a peer node allocated out of its own slice. The `test-run` suites + // give each node a /24 of its own, so a peer's address // is outside this node's pool *and* outside its interface network — yet // every CVM is handed every gateway as a WireGuard server, so this node // still has to carry it. diff --git a/dstack/gateway/test-run/cluster.sh b/dstack/gateway/test-run/cluster.sh deleted file mode 100755 index 27f58b4c6..000000000 --- a/dstack/gateway/test-run/cluster.sh +++ /dev/null @@ -1,441 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Gateway cluster management script for manual testing - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -GATEWAY_BIN="${SCRIPT_DIR}/../../target/release/dstack-gateway" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -TMUX_SESSION="gateway-cluster" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -show_help() { - echo "Gateway Cluster Management Script" - echo "" - echo "Usage: $0 " - echo "" - echo "Commands:" - echo " start Start a 3-node gateway cluster in tmux" - echo " stop Stop the cluster (keep tmux session)" - echo " reg Register a random instance" - echo " status Show cluster status" - echo " clean Destroy cluster and clean all data" - echo " attach Attach to tmux session" - echo " help Show this help" - echo "" -} - -# Generate certificates -generate_certs() { - mkdir -p "$CERTS_DIR" - mkdir -p "$RUN_DIR/certbot/live" - - # Generate CA certificate - if [[ ! -f "$CERTS_DIR/gateway-ca.key" ]]; then - log_info "Creating CA certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-ca.key" 2048 2>/dev/null - openssl req -x509 -new -nodes \ - -key "$CERTS_DIR/gateway-ca.key" \ - -sha256 -days 365 \ - -out "$CERTS_DIR/gateway-ca.cert" \ - -subj "/CN=Test CA/O=Gateway Test" \ - 2>/dev/null - fi - - # Generate RPC certificate signed by CA - if [[ ! -f "$CERTS_DIR/gateway-rpc.key" ]]; then - log_info "Creating RPC certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-rpc.key" 2048 2>/dev/null - openssl req -new \ - -key "$CERTS_DIR/gateway-rpc.key" \ - -out "$CERTS_DIR/gateway-rpc.csr" \ - -subj "/CN=localhost" \ - 2>/dev/null - cat > "$CERTS_DIR/ext.cnf" << EXTEOF -authorityKeyIdentifier=keyid,issuer -basicConstraints=CA:FALSE -keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = localhost -IP.1 = 127.0.0.1 -EXTEOF - openssl x509 -req \ - -in "$CERTS_DIR/gateway-rpc.csr" \ - -CA "$CERTS_DIR/gateway-ca.cert" \ - -CAkey "$CERTS_DIR/gateway-ca.key" \ - -CAcreateserial \ - -out "$CERTS_DIR/gateway-rpc.cert" \ - -days 365 \ - -sha256 \ - -extfile "$CERTS_DIR/ext.cnf" \ - 2>/dev/null - rm -f "$CERTS_DIR/gateway-rpc.csr" "$CERTS_DIR/ext.cnf" - fi - - # Generate proxy certificates - local proxy_cert_dir="$RUN_DIR/certbot/live" - if [[ ! -f "$proxy_cert_dir/cert.pem" ]]; then - log_info "Creating proxy certificates..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$proxy_cert_dir/key.pem" \ - -out "$proxy_cert_dir/cert.pem" \ - -days 365 \ - -subj "/CN=localhost" \ - 2>/dev/null - fi - - # Generate unique WireGuard key pair for each node - for i in 1 2 3; do - if [[ ! -f "$CERTS_DIR/wg-node${i}.key" ]]; then - log_info "Generating WireGuard keys for node ${i}..." - wg genkey > "$CERTS_DIR/wg-node${i}.key" - wg pubkey < "$CERTS_DIR/wg-node${i}.key" > "$CERTS_DIR/wg-node${i}.pub" - fi - done -} - -# Generate node config -generate_config() { - local node_id=$1 - local rpc_port=$((13000 + node_id * 10 + 2)) - local wg_port=$((13000 + node_id * 10 + 3)) - local proxy_port=$((13000 + node_id * 10 + 4)) - local debug_port=$((13000 + node_id * 10 + 5)) - local admin_port=$((13000 + node_id * 10 + 6)) - local wg_ip="10.0.3${node_id}.1/24" - local other_nodes="" - local peer_urls="" - - # Read WireGuard keys for this node - local wg_private_key=$(cat "$CERTS_DIR/wg-node${node_id}.key") - local wg_public_key=$(cat "$CERTS_DIR/wg-node${node_id}.pub") - - for i in 1 2 3; do - if [[ $i -ne $node_id ]]; then - local peer_rpc_port=$((13000 + i * 10 + 2)) - if [[ -n "$other_nodes" ]]; then - other_nodes="$other_nodes, $i" - peer_urls="$peer_urls, \"$i:https://localhost:$peer_rpc_port\"" - else - other_nodes="$i" - peer_urls="\"$i:https://localhost:$peer_rpc_port\"" - fi - fi - done - - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - cat > "$RUN_DIR/node${node_id}.toml" << EOF -log_level = "info" -address = "0.0.0.0" -port = ${rpc_port} - -[tls] -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -rpc_domain = "" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -port = ${debug_port} -address = "127.0.0.1" - -[core.admin] -enabled = true -port = ${admin_port} -address = "127.0.0.1" - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_node${node_id}" - -[core.certbot] -enabled = false - -[core.wg] -private_key = "${wg_private_key}" -public_key = "${wg_public_key}" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.3${node_id}.1/31"] -client_ip_range = "10.0.3${node_id}.1/24" -config_path = "${RUN_DIR}/wg_node${node_id}.conf" -interface = "gw-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${RUN_DIR}/certbot/live/cert.pem" -cert_key = "${RUN_DIR}/certbot/live/key.pem" -base_domain = "test.local" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} - -[core.recycle] -enabled = true -interval = "30s" -timeout = "120s" -node_timeout = "300s" -EOF -} - -# Build gateway binary -build_gateway() { - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_info "Building gateway..." - (cd "$SCRIPT_DIR/.." && cargo build --release) - fi -} - -# Start cluster -cmd_start() { - build_gateway - generate_certs - - # Check if tmux session exists - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - log_warn "Cluster already running. Use 'clean' to restart." - cmd_status - return 0 - fi - - log_info "Generating configs..." - mkdir -p "$RUN_DIR" "$LOG_DIR" - for i in 1 2 3; do - generate_config $i - mkdir -p "$RUN_DIR/wavekv_node${i}" - done - - log_info "Starting cluster in tmux session '$TMUX_SESSION'..." - - # Create wrapper scripts that keep running even if gateway exits - for i in 1 2 3; do - cat > "$RUN_DIR/run_node${i}.sh" << RUNEOF -#!/bin/bash -cd "$SCRIPT_DIR" -while true; do - echo "Starting node ${i}..." - sudo RUST_LOG=info $GATEWAY_BIN -c $RUN_DIR/node${i}.toml 2>&1 | tee -a $LOG_DIR/node${i}.log - echo "Node ${i} exited. Press Ctrl+C to stop, or wait 3s to restart..." - sleep 3 -done -RUNEOF - chmod +x "$RUN_DIR/run_node${i}.sh" - done - - # Create tmux session - tmux new-session -d -s "$TMUX_SESSION" -n "node1" - tmux send-keys -t "$TMUX_SESSION:node1" "$RUN_DIR/run_node1.sh" Enter - - sleep 1 - - # Add windows for other nodes - tmux new-window -t "$TMUX_SESSION" -n "node2" - tmux send-keys -t "$TMUX_SESSION:node2" "$RUN_DIR/run_node2.sh" Enter - - tmux new-window -t "$TMUX_SESSION" -n "node3" - tmux send-keys -t "$TMUX_SESSION:node3" "$RUN_DIR/run_node3.sh" Enter - - # Add a shell window - tmux new-window -t "$TMUX_SESSION" -n "shell" - - sleep 3 - - log_info "Cluster started!" - echo "" - cmd_status - echo "" - log_info "Use '$0 attach' to view logs" -} - -# Stop cluster -cmd_stop() { - log_info "Stopping cluster..." - sudo pkill -9 -f "dstack-gateway.*node[123].toml" 2>/dev/null || true - sudo ip link delete gw-test1 2>/dev/null || true - sudo ip link delete gw-test2 2>/dev/null || true - sudo ip link delete gw-test3 2>/dev/null || true - log_info "Cluster stopped" -} - -# Clean everything -cmd_clean() { - cmd_stop - - # Kill tmux session - tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true - - log_info "Cleaning data..." - sudo rm -rf "$RUN_DIR/wavekv_node"* - sudo rm -f "$RUN_DIR/gateway-state-node"*.json - rm -f "$RUN_DIR/wg_node"*.conf - rm -f "$RUN_DIR/node"*.toml - rm -f "$RUN_DIR/run_node"*.sh - rm -rf "$LOG_DIR" - - log_info "Cleaned" -} - -# Show status -cmd_status() { - echo -e "${BLUE}=== Gateway Cluster Status ===${NC}" - echo "" - - for i in 1 2 3; do - local rpc_port=$((13000 + i * 10 + 2)) - local proxy_port=$((13000 + i * 10 + 4)) - local debug_port=$((13000 + i * 10 + 5)) - local admin_port=$((13000 + i * 10 + 6)) - - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - echo -e "Node $i: ${GREEN}RUNNING${NC}" - else - echo -e "Node $i: ${RED}STOPPED${NC}" - fi - echo " RPC: https://localhost:${rpc_port}" - echo " Proxy: https://localhost:${proxy_port}" - echo " Debug: http://localhost:${debug_port}" - echo " Admin: http://localhost:${admin_port}" - echo "" - done - - # Show instance count from first running node - for i in 1 2 3; do - local debug_port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - local response=$(curl -s -X POST "http://localhost:${debug_port}/prpc/GetSyncData" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null) - if [[ -n "$response" ]]; then - local n_instances=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('instances', [])))" 2>/dev/null || echo "?") - local n_nodes=$(echo "$response" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('nodes', [])))" 2>/dev/null || echo "?") - echo -e "${BLUE}Cluster State:${NC}" - echo " Nodes: $n_nodes" - echo " Instances: $n_instances" - fi - break - fi - done -} - -# Register a random instance -cmd_reg() { - # Find a running node - local debug_port="" - for i in 1 2 3; do - local port=$((13000 + i * 10 + 5)) - if pgrep -f "dstack-gateway.*node${i}.toml" > /dev/null 2>&1; then - debug_port=$port - break - fi - done - - if [[ -z "$debug_port" ]]; then - log_error "No running nodes found. Start cluster first." - exit 1 - fi - - # Generate random WireGuard key pair - local private_key=$(wg genkey) - local public_key=$(echo "$private_key" | wg pubkey) - - # Generate random IDs - local app_id="app-$(openssl rand -hex 4)" - local instance_id="inst-$(openssl rand -hex 4)" - - log_info "Registering instance..." - log_info " App ID: $app_id" - log_info " Instance ID: $instance_id" - log_info " Public Key: $public_key" - - local response=$(curl -s \ - -X POST "http://localhost:${debug_port}/prpc/RegisterCvm" \ - -H "Content-Type: application/json" \ - -d "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" 2>/dev/null) - - if echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'wg' in d" 2>/dev/null; then - local client_ip=$(echo "$response" | python3 -c "import sys,json; print(json.load(sys.stdin)['wg']['client_ip'])" 2>/dev/null) - log_info "Registered successfully!" - echo -e " Client IP: ${GREEN}$client_ip${NC}" - echo "" - echo "Instance details:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - else - log_error "Registration failed:" - echo "$response" | python3 -m json.tool 2>/dev/null || echo "$response" - exit 1 - fi -} - -# Attach to tmux -cmd_attach() { - if tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then - tmux attach -t "$TMUX_SESSION" - else - log_error "No cluster running" - exit 1 - fi -} - -# Main -case "${1:-help}" in - start) - cmd_start - ;; - stop) - cmd_stop - ;; - clean) - cmd_clean - ;; - status) - cmd_status - ;; - reg) - cmd_reg - ;; - attach) - cmd_attach - ;; - help|--help|-h) - show_help - ;; - *) - log_error "Unknown command: $1" - show_help - exit 1 - ;; -esac diff --git a/dstack/gateway/test-run/cluster/docker-compose.yml b/dstack/gateway/test-run/cluster/docker-compose.yml new file mode 100644 index 000000000..bd4b6984c --- /dev/null +++ b/dstack/gateway/test-run/cluster/docker-compose.yml @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# WaveKV / cluster integration suite. +# +# Driven from the host: the driver rewrites a node's config and stops or starts +# its container between tests, which is what the ported tests need and what a +# runner inside the network could not do without the docker socket. +# +# Each node's config and data directory are bind mounts, not tmpfs: the tests +# read and write node state directly -- 17 of 28 do -- and four perform surgery +# on a stopped node's data directory (wipe it but keep node_uuid, stat the WAL). +# +# Host ports are left for docker to assign and the driver looks them up with +# `docker compose port`. Fixed host ports are a standing collision with whatever +# else the developer is running -- the suite this replaces squatted on 13012-13036 +# and carried a wait-for-the-port-to-free loop because of it. +# +# Config and data live under the test's own name, so a test cannot read or +# write what another one left. CURRENT_TEST is exported by the driver before it +# brings a project up; the default keeps `docker compose` usable by hand. +# +# The config is mounted as a DIRECTORY rather than a file. The driver rewrites +# node configs between tests, and a single-file bind mount pins the original +# inode, so a rewritten file would never be seen inside the container. + +# The fixture is a long-lived project of its own, started once by the driver. +# Each test then gets a throwaway project that joins the fixture's network, so a +# test's containers, logs and data start empty because they are new rather than +# because something cleaned them. +networks: + attestation: + external: true + name: dstack-attestation + +volumes: + dstack-socket: + external: true + name: dstack-attestation-socket + attestation-roots: + external: true + name: dstack-attestation-roots + +services: + gateway-1: &node + image: ${GATEWAY_IMAGE:-dstack-gateway:test} + networks: + attestation: + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node1:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node1:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + environment: + - RUST_LOG=info,dstack_gateway=debug + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + cap_add: + - NET_ADMIN + + gateway-2: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node2:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node2:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack + + gateway-3: + <<: *node + ports: + - "9012" + - "9015" + - "9016" + volumes: + - ./run/configs/${CURRENT_TEST:-suite}/node3:/etc/gateway + - ./run/data/${CURRENT_TEST:-suite}/node3:/var/lib/gateway + - attestation-roots:/var/lib/attestation:ro + - dstack-socket:/var/run/dstack diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh new file mode 100644 index 000000000..69005403b --- /dev/null +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Host-side driver primitives for the gateway cluster suite. +# +# The tests need to stop a node, rewrite its config, wipe or doctor its data +# directory and start it again. All of that happens here, on the host, against +# bind mounts and `docker compose`. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="$SCRIPT_DIR/run" +CONFIG_DIR="$RUN_DIR/configs" +DATA_DIR="$RUN_DIR/data" +LOG_DIR="$RUN_DIR/logs" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { printf "${BLUE}[INFO]${NC} %s\n" "$1" >&2; } +log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1" >&2; } +log_error() { printf "${RED}[ERROR]${NC} %s\n" "$1" >&2; } +log_success() { printf "${GREEN}[PASS]${NC} %s\n" "$1" >&2; } +log_fail() { printf "${RED}[FAIL]${NC} %s\n" "$1" >&2; } + +# Host-side ports are assigned by docker, not fixed, so the suite cannot +# collide with whatever else is listening on the machine. Look them up once a +# container is running and cache the answer -- `docker compose port` is a +# process spawn, and the tests ask for these constantly. +declare -A _PORT_CACHE=() +container_port() { + local node_id=$1 + local container_port=$2 + # Keyed by project too: each test gets its own containers, so a port + # cached under a previous test's project points at something that no longer + # exists -- which reads as "the node did not come up". + local key="${CURRENT_TEST:-suite}:${node_id}:${container_port}" + local mapped + if [[ -n "${_PORT_CACHE[$key]:-}" ]]; then + echo "${_PORT_CACHE[$key]}"; return 0 + fi + mapped=$(compose port "gateway-${node_id}" "$container_port" 2>/dev/null | awk -F: 'NF{print $NF}') + [[ -n "$mapped" ]] || return 1 + _PORT_CACHE[$key]="$mapped" + echo "$mapped" +} + +# A restarted container gets a new host port, so anything that stops a node has +# to drop the cache or later lookups address a port nothing is listening on. +forget_ports() { + local node_id=$1 + local t="${CURRENT_TEST:-suite}" + unset '_PORT_CACHE['"$t:$node_id"':9012]' \ + '_PORT_CACHE['"$t:$node_id"':9015]' \ + '_PORT_CACHE['"$t:$node_id"':9016]' +} + +rpc_port() { container_port "$1" 9012; } +debug_port() { container_port "$1" 9015; } +admin_port() { container_port "$1" 9016; } + +# Write node $1's config, wiring its bootnode to $2 (empty for none). +# +# Everything except the bootnode is derived from the node id, which is why the +# original suite could regenerate this 30-odd times and only ever change one +# field. +generate_config() { + local node_id=$1 + local bootnode_url=${2:-""} + local dir="$CONFIG_DIR/${CURRENT_TEST:-suite}/node${node_id}" + mkdir -p "$dir" + cat >"$dir/gateway.toml" </dev/null +} + +# Each test runs in a compose project of its own, so its containers, its logs +# and its data start empty because they are new -- not because something cleaned +# them. CURRENT_TEST is set by the runner before each test. +compose() { + docker compose -p "cluster-${CURRENT_TEST:-suite}" \ + -f "$SCRIPT_DIR/docker-compose.yml" "$@" +} + +# The attestation fixture is a separate, long-lived project; the runner owns it. +fixture_compose() { + docker compose -p dstack-fixture \ + -f "$SCRIPT_DIR/../attestation/fixture.yml" "$@" +} + +# Discard whatever the previous test used and start this one from nothing. +new_test_project() { + compose down -v --remove-orphans >/dev/null 2>&1 || true + _PORT_CACHE=() + CURRENT_TEST="$1" + export CURRENT_TEST + mkdir -p "$DATA_DIR/$CURRENT_TEST" +} + +start_node() { + local node_id=$1 + # `compose start` exits 0 when the service has no container yet -- it prints + # "has no container to start" and succeeds -- so an `||` fallback to + # `compose up -d` never fires. Under one project per test the container + # usually does not exist, which made every node after the first look like it + # failed to come up. `up -d` both creates and starts, and is a no-op for a + # container that is already running. + compose up -d "gateway-${node_id}" >/dev/null 2>&1 || + compose start "gateway-${node_id}" >/dev/null 2>&1 + forget_ports "$node_id" + wait_for_debug "$node_id" 60 || { log_error "node ${node_id} did not come up"; return 1; } +} + +stop_node() { + local node_id=$1 + compose stop -t 5 "gateway-${node_id}" >/dev/null 2>&1 || true + forget_ports "$node_id" +} + +wait_for_debug() { + local node_id=$1 + local timeout=${2:-60} + local waited=0 + local port + while [ "$waited" -lt "$timeout" ]; do + # Resolved inside the loop on purpose: docker only reports the mapping + # once the container is running, so looking it up once up front would + # leave every later request aimed at an empty port. + if port=$(debug_port "$node_id") && [ -n "$port" ] && + curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.Info" \ + -H 'Content-Type: application/json' -d '{}' >/dev/null 2>&1; then + return 0 + fi + sleep 1; waited=$((waited + 1)) + done + return 1 +} + +# Two ported tests assert on what a node logged. Containers keep that in the +# daemon rather than a file, so materialise it on demand and leave the +# assertions themselves unchanged. +# +# $2 bounds the window. The suite it replaces wrote a fresh log file per test, +# so a grep could not see anything an earlier test produced; `docker logs` +# accumulates for the life of the container, and without a bound a test looking +# for a message an earlier one legitimately caused would pass without that +# message ever being emitted again. +dump_log() { + local node_id=$1 + mkdir -p "$LOG_DIR" + # No time window. This container was created for this test, so everything in + # its log belongs to this test -- and the window the shared-container design + # needed is what made the docker-logs timezone bug possible. + compose logs --no-color "gateway-${node_id}" \ + >"$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" 2>&1 || true + echo "$LOG_DIR/${CURRENT_TEST}-node${node_id}.log" +} + + +node_data_dir() { echo "$DATA_DIR/${CURRENT_TEST:-suite}/node$1"; } + +# The gateway runs as root inside its container, so everything it writes into +# the bind-mounted data directory is root-owned. Reading it from the host is +# fine -- node_uuid and the WAL are world-readable, which is all the surgery +# tests need -- but removing or replacing it is not. Do that in a throwaway +# root container rather than reaching for sudo, which is exactly the dependency +# this port exists to shed. +# +# $1 is a shell snippet run against /data, the parent of the per-node dirs. +data_op() { + docker run --rm -v "$DATA_DIR/${CURRENT_TEST}:/data" alpine:latest sh -c "$1" +} + +# Wipe a stopped node's store but let it keep the identity it already published, +# which is what a node looks like after losing its disk but not its config. +wipe_data_keeping_uuid() { + local node_id=$1 + data_op "cp /data/node${node_id}/wavekv/node_uuid /tmp/uuid && + rm -rf /data/node${node_id}/wavekv/* /data/node${node_id}/wavekv/.[!.]* 2>/dev/null; + cp /tmp/uuid /data/node${node_id}/wavekv/node_uuid" +} + +# Wipe a stopped node completely, identity included -- a stranger turning up +# with a node id that is already taken. Contents only: the directory itself is a +# bind-mount source, and replacing it swaps the inode the mount was set against. +wipe_data() { + local node_id=$1 + data_op "rm -rf /data/node${node_id}/..?* /data/node${node_id}/.[!.]* /data/node${node_id}/*" \ + 2>/dev/null || true +} diff --git a/dstack/gateway/test-run/cluster/rpc.sh b/dstack/gateway/test-run/cluster/rpc.sh new file mode 100644 index 000000000..0c125f08e --- /dev/null +++ b/dstack/gateway/test-run/cluster/rpc.sh @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# RPC helpers for the cluster suite. +# +# These take a node id rather than a port. The process-based suite passed ports +# around because they were fixed and computable; here docker assigns them, so +# resolving one is a lookup and every caller would otherwise have to do it. + +debug_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(debug_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +admin_call() { + local node_id=$1 + local method=$2 + local body=${3:-'{}'} + local port + port=$(admin_port "$node_id") || return 1 + curl -s -X POST "http://127.0.0.1:${port}/prpc/${method}" \ + -H "Content-Type: application/json" -d "$body" 2>/dev/null +} + +json_field() { python3 -c "$1" 2>/dev/null; } + +check_debug_service() { + debug_call "$1" Debug.Info \ + | json_field "import sys,json; d=json.load(sys.stdin); assert 'base_domain' in d" +} + +debug_get_sync_data() { debug_call "$1" Debug.GetSyncData; } + +# A WireGuard public key is 32 bytes; derive a distinct one per seed so +# registrations do not collide. +test_public_key() { + python3 -c "import base64; print(base64.b64encode(int($1).to_bytes(32, 'big')).decode())" +} + +debug_register_cvm() { + local node_id=$1 + local public_key=$2 + local app_id=${3:-testapp} + local instance_id=${4:-testinstance} + debug_call "$node_id" RegisterCvm \ + "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" +} + +# Prints the allocated client_ip, or fails. +verify_register_response() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) + assert 'wg' in d, 'missing wg config' + assert 'client_ip' in d['wg'], 'missing client_ip' + print(d['wg']['client_ip']) +except Exception as e: + print(f'ERROR: {e}', file=sys.stderr) + sys.exit(1) +" 2>/dev/null +} + +_sync_data_has() { + local node_id=$1 + local collection=$2 + local peer_node_id=$3 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for entry in d.get('$collection', []): + if entry.get('node_id') == $peer_node_id: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} + +has_peer_addr() { _sync_data_has "$1" peer_addrs "$2"; } +has_node_info() { _sync_data_has "$1" nodes "$2"; } + +get_n_instances() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +get_n_nodes() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('nodes', []))) +except Exception: + print(0) +" 2>/dev/null +} + +# Two different endpoints, and conflating them silently breaks whatever reads +# the wrong one: WaveKvStatus carries the store digests and key counts, Status +# carries node identity and the peer list. +admin_wavekv_status() { admin_call "$1" Admin.WaveKvStatus; } +admin_status() { admin_call "$1" Admin.Status; } + +get_store_digest() { + local node_id=$1 + local store=$2 + admin_wavekv_status "$node_id" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['$store']['digest'])" 2>/dev/null || true +} + +# A node's own uuid, as it reports itself in its node list. +get_node_uuid() { + local node_id=$1 + admin_status "$node_id" | python3 -c " +import sys, json +d = json.load(sys.stdin) +me = d.get('id') +for n in d.get('nodes', []): + if n.get('id') == me: + print(n.get('uuid', '')) + break +" 2>/dev/null || true +} + +admin_set_node_url() { + admin_call "$1" Admin.SetNodeUrl "{\"id\": $2, \"url\": \"$3\"}" +} + +# Peers address each other by container hostname, not by a published port: the +# gateways talk over the compose network, and only the driver goes through the +# host. +setup_peers() { + local node_ids=("$@") + local src dst + for src in "${node_ids[@]}"; do + for dst in "${node_ids[@]}"; do + [ "$src" = "$dst" ] && continue + admin_set_node_url "$src" "$dst" "https://gateway-${dst}:9012" >/dev/null + done + done +} + +wait_for_instances() { + local node_id=$1 + local expected=$2 + local timeout_seconds=$3 + local _ + for _ in $(seq 1 $((timeout_seconds * 10))); do + [ "$(get_n_instances "$node_id")" -ge "$expected" ] 2>/dev/null && return 0 + sleep 0.1 + done + return 1 +} + +wait_for_digest_match() { + local store=$1 + local node_a=$2 + local node_b=$3 + local timeout_seconds=$4 + local d1 d2 _ + for _ in $(seq 1 $((timeout_seconds * 10))); do + d1=$(get_store_digest "$node_a" "$store") + d2=$(get_store_digest "$node_b" "$store") + if [ -n "$d1" ] && [ "$d1" = "$d2" ]; then return 0; fi + sleep 0.1 + done + return 1 +} + +get_n_keys() { + admin_wavekv_status "$1" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['persistent']['n_keys'])" 2>/dev/null || echo 0 +} + +# The gateway writes its CA where only root can read it, so copy it out for the +# one test that validates the RPC chain rather than skipping verification. +export_ca_cert() { + local node_id=$1 + local dest="$RUN_DIR/node${node_id}-ca.cert" + data_op "cat /data/node${node_id}/certs/gateway-ca.cert" >"$dest" 2>/dev/null + [ -s "$dest" ] || return 1 + echo "$dest" +} + +get_n_peer_addrs() { + debug_get_sync_data "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('peer_addrs', []))) +except Exception: + print(0) +" 2>/dev/null +} + +debug_get_proxy_state() { debug_call "$1" GetProxyState; } + +get_n_proxy_state_instances() { + debug_get_proxy_state "$1" | python3 -c " +import sys, json +try: + print(len(json.load(sys.stdin).get('instances', []))) +except Exception: + print(0) +" 2>/dev/null +} + +admin_set_node_status() { + admin_call "$1" Admin.SetNodeStatus "{\"id\": $2, \"status\": \"$3\"}" +} + +get_peer_url_from_sync() { + local node_id=$1 + local peer_node_id=$2 + debug_get_sync_data "$node_id" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for pa in d.get('peer_addrs', []): + if pa.get('node_id') == $peer_node_id: + print(pa.get('url', '')) + sys.exit(0) + print('') +except Exception: + print('') +" 2>/dev/null +} + +# Whether a RegisterCvm response offered the CVM a given gateway. +response_lists_gateway() { + echo "$1" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) + for gw in d.get('gateways', []): + if gw.get('id') == $2: + sys.exit(0) + sys.exit(1) +except Exception: + sys.exit(1) +" +} diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh new file mode 100755 index 000000000..563f91723 --- /dev/null +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Gateway cluster (WaveKV) integration suite, driven from the host. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" +# shellcheck source=rpc.sh +source "$SCRIPT_DIR/rpc.sh" +# shellcheck source=tests.sh +source "$SCRIPT_DIR/tests.sh" + +SKIP_BUILD="" +KEEP_RUNNING=false +ONLY="" +while [[ $# -gt 0 ]]; do + case $1 in + --skip-build) SKIP_BUILD="--skip-build"; shift ;; + --keep-running) KEEP_RUNNING=true; shift ;; + --only) ONLY="$2"; shift 2 ;; + down) + compose down -v --remove-orphans 2>/dev/null || true + exit 0 ;; + -h|--help) + echo "Usage: $0 [--skip-build] [--keep-running] [--only ] | down" + exit 0 ;; + *) log_error "unknown option: $1"; exit 1 ;; + esac +done + +# Two runs of this suite share one compose project, one set of container names +# and one data directory, so a second run stops the containers the first is +# using and wipes the store out from under it. The result does not look like a +# collision -- it looks like flaky cross-node sync, because a node's writes +# vanish mid-test. Refuse to start instead. +# The lock is taken on the run directory itself, not on a file inside it. +# Deleting a lock FILE does not release the lock -- it detaches it: the holder +# keeps its inode while the next run creates a fresh one and flocks that +# successfully. Three runs of this suite overlapped that way, each wiping the +# others' state, and the failures read as flaky cross-node sync rather than as +# a collision. A directory is not something a cleanup step removes by name, and +# `rm -rf run/data run/configs run/logs` leaves it in place. +mkdir -p "$RUN_DIR" +exec 9<"$RUN_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($RUN_DIR)" + exit 1 +fi + +# No process-name guard beyond the lock. One was tried and false-positived on +# its own first run: `$(...)` forks a subshell whose argv is identical to the +# script's, so `pgrep -f` counted the check itself as a second instance. The +# directory lock above is the correct primitive -- it is race-free and needs no +# pattern matching. + +cleanup() { + $KEEP_RUNNING && return 0 + # The last test's project is still up; the fixture outlives them all. + compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi +} +trap cleanup EXIT + +TESTS_PASSED=0 +TESTS_FAILED=0 +check() { + local name="$1"; shift + if "$@"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + fi +} + +# ---------------------------------------------------------------- assertions + +# Every node must see all three, on every node. +cluster_has_three_nodes() { + local node_id=$1 port count + port=$(debug_port "$node_id") + count=$(curl -sf -X POST "http://127.0.0.1:${port}/prpc/Debug.GetSyncData" \ + -H 'Content-Type: application/json' -d '{}' 2>/dev/null \ + | python3 -c 'import sys,json; print(len(json.load(sys.stdin).get("nodes",[])))' 2>/dev/null) || return 1 + [ "$count" = "3" ] +} + +# Positive evidence that the cluster's mTLS is real: the certificate a node +# serves carries the app_id extension its peers pin, which only happens when it +# was issued through the guest agent against a verifiable quote. Absence of +# errors would not prove this -- a validator that never ran is also silent. +cert_carries_app_id() { + local node_id=$1 + data_op "cat /data/node${node_id}/certs/gateway-rpc.cert" 2>/dev/null \ + | openssl x509 -noout -text 2>/dev/null \ + | grep -q "1.3.6.1.4.1.62397.1.3" +} + +# And the negative: no peer was ever turned away. +# +# Deliberately not matching "bootnode discovery retry failed". A node whose +# bootnode is not listening yet retries and succeeds, which is a startup race, +# not an authentication failure -- treating it as one made this assertion fail +# on whichever node happened to start first. Convergence is what proves the +# retry worked, and that is asserted separately. +no_peer_was_rejected() { + local node_id=$1 log + log=$(dump_log "$node_id") + ! grep -qE "does not contain app_id|invalid quote|app_id mismatch" "$log" +} + +# ---------------------------------------------------------------- main + +log_info "==========================================" +log_info "dstack-gateway cluster suite" +log_info "==========================================" + +rm -rf "$LOG_DIR" +mkdir -p "$DATA_DIR" + +log_info "starting the attestation fixture" +fixture_compose build >/dev/null +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) +fixture_compose up -d --wait >/dev/null + +"$SCRIPT_DIR/../build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD + +log_info "building the suite image" +compose build >/dev/null + +# The smoke checks get a project of their own for the same reason every test +# does: the first test's new_test_project tears down whatever came before it, +# and without this the smoke nodes were what it tore down. +# +# It has to come before the configs are written: config and data paths are named +# after CURRENT_TEST, so generating them first puts them where nothing looks. +new_test_project smoke + +log_info "generating node configs" +generate_config 1 "" +generate_config 2 "https://gateway-1:9012" +generate_config 3 "https://gateway-1:9012" + +log_info "starting the cluster" +compose up -d >/dev/null + +for n in 1 2 3; do + check "node $n debug service is up" wait_for_debug "$n" 90 +done + +log_info "waiting for the cluster to converge" +converged=false +for _ in $(seq 1 30); do + if cluster_has_three_nodes 1 && cluster_has_three_nodes 2 && cluster_has_three_nodes 3; then + converged=true; break + fi + sleep 2 +done +$converged || log_warn "cluster did not converge within 60s" + +for n in 1 2 3; do + check "node $n sees all three nodes" cluster_has_three_nodes "$n" + check "node $n serves a certificate carrying its app_id" cert_carries_app_id "$n" + check "node $n rejected no peer" no_peer_was_rejected "$n" +done + +# ------------------------------------------------------------- ported tests +# +# Every row here must also be ticked in the port checklist only once it has been +# seen to FAIL against a broken cluster. Green on its own proves nothing. +run_ported() { + local name="$1" + log_info "---------- $name ----------" + new_test_project "$name" + if "$name"; then + log_success "$name"; TESTS_PASSED=$((TESTS_PASSED + 1)) + else + log_fail "$name"; TESTS_FAILED=$((TESTS_FAILED + 1)) + # Under one project per test the next test's new_test_project tears this + # one down, so a failure's container logs are gone before anyone can + # read them. Dumping them here is what makes a failure diagnosable at + # all: test_partial_cluster_bootstrap failed with nothing in the suite + # log but its own config lines, and there was no way to tell which step + # had failed. An explicit `if` rather than `A && B`, because SC2015 has + # already made a successful run exit non-zero once. + for _n in 1 2 3; do + if compose ps -q "gateway-${_n}" 2>/dev/null | grep -q .; then + log_info " saved log: $(dump_log "$_n")" + fi + done + fi +} + +ALL_TESTS=(test_persistence test_status_endpoint test_prpc_register \ + test_prpc_info test_wal_integrity \ + test_multi_node_sync test_node_recovery test_cross_node_data_sync \ + test_push_fast_path test_periodic_repair_after_missed_push \ + test_bootstrap_after_data_dir_loss test_divergent_partition_writes \ + test_push_periodic_overlap test_delayed_bootnode_recovery \ + test_interrupted_sync_recovery test_ephemeral_recovery \ + test_partial_cluster_bootstrap test_node_id_reuse_rejected \ + test_client_registration_persistence test_stress_writes \ + test_network_partition test_three_node_cluster \ + test_three_node_bootnode test_periodic_persistence \ + test_admin_set_node_url test_admin_set_node_status \ + test_node_status_register_exclude test_node_status_register_reject) + +for t in "${ALL_TESTS[@]}"; do + [ -n "$ONLY" ] && [ "$t" != "$ONLY" ] && continue + run_ported "$t" +done + +log_info "==========================================" +log_info "Passed: $TESTS_PASSED" +log_info "Failed: $TESTS_FAILED" +log_info "==========================================" +[ "$TESTS_FAILED" -eq 0 ] diff --git a/dstack/gateway/test-run/cluster/tests.sh b/dstack/gateway/test-run/cluster/tests.sh new file mode 100644 index 000000000..a6f590df5 --- /dev/null +++ b/dstack/gateway/test-run/cluster/tests.sh @@ -0,0 +1,816 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash + +# Ported from test-run/test_suite.sh. Each test keeps the assertions it had; +# what changed is how nodes are started, stopped and reset. + +# Nothing to clean: the runner gave this test a compose project of its own, so +# its containers and its data directory are new. Kept as a no-op so each test +# still reads as declaring where it starts from. +cleanup_cluster() { :; } + +# ------------------------------------------------------------------ quick + +test_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + sleep 2 + local keys_after_write + keys_after_write=$(get_n_keys 1) + log_info "keys after startup: $keys_after_write" + + stop_node 1 + start_node 1 || return 1 + + local keys_after_restart + keys_after_restart=$(get_n_keys 1) + log_info "keys after restart: $keys_after_restart" + + # get_n_keys ends in `|| echo 0`, so an unreachable node reports zero keys + # rather than failing. Without this guard a node that never wrote anything + # gives 0 >= 0 and the test passes having proved nothing -- the same shape + # as the empty-store digest match that made test_ephemeral_recovery hollow. + [ "${keys_after_write:-0}" -gt 0 ] 2>/dev/null || { + log_error "nothing was in the store before the restart, so surviving it proves nothing: keys=${keys_after_write:-0}" + return 1; } + if [ "$keys_after_restart" -ge "$keys_after_write" ] 2>/dev/null; then + return 0 + fi + log_error "expected >= $keys_after_write keys, got $keys_after_restart" + return 1 +} + +test_status_endpoint() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + admin_wavekv_status 1 | python3 -c " +import sys, json +d = json.load(sys.stdin) +assert d['enabled'] is True, 'enabled should be True' +assert 'persistent' in d, 'missing persistent' +assert 'ephemeral' in d, 'missing ephemeral' +assert d['persistent']['wal_enabled'] is True, 'persistent wal should be enabled' +assert d['ephemeral']['wal_enabled'] is False, 'ephemeral wal should be disabled' +assert 'peers' in d['persistent'], 'missing peers in persistent' +" +} + +test_prpc_register() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 501)" deadbeef cafebabe) + verify_register_response "$response" >/dev/null || { + log_error "RegisterCvm did not return a client_ip"; return 1; } +} + +test_prpc_info() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + local ca port + ca=$(export_ca_cert 1) || { log_error "could not read the node's CA certificate"; return 1; } + port=$(rpc_port 1) || return 1 + + curl -s --cacert "$ca" --resolve "gateway-1:${port}:127.0.0.1" \ + -X POST "https://gateway-1:${port}/prpc/Info" \ + -H "Content-Type: application/json" -d '{}' 2>/dev/null | python3 -c " +import sys, json +d = json.load(sys.stdin) +if 'error' in d: + print(f'ERROR: {d[\"error\"]}', file=sys.stderr) + sys.exit(1) +assert 'base_domain' in d, 'missing base_domain' +assert 'external_port' in d, 'missing external_port' +" +} + +test_wal_integrity() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key response success=0 + for i in $(seq 1 5); do + key=$(test_public_key $((800 + i))) + response=$(debug_register_cvm 1 "$key" "wal_app$i" "wal_inst$i") + verify_register_response "$response" >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 5 ] || { log_error "registered only $success/5 clients"; return 1; } + + # The WAL has to exist and be non-empty: a store that silently stopped + # journalling would still answer every RPC above. + local wal_size + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + log_info "WAL size: ${wal_size} bytes" + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null +} + +# ------------------------------------------------------------------- sync + +# Two nodes must learn each other's address and node record, in both directions. +test_multi_node_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 10 + + local ok=0 + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; ok=1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; ok=1; } + has_node_info 1 2 || { log_error "node 1 missing node_info for node 2"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1"; ok=1; } + return $ok +} + +# A node that was down has to catch up once it is back. +test_node_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 5 + + stop_node 2 + sleep 3 + start_node 2 || return 1 + setup_peers 1 2 + sleep 10 + + local ok=0 + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1 after recovery"; ok=1; } + has_node_info 2 1 || { log_error "node 2 missing node_info for node 1 after recovery"; ok=1; } + return $ok +} + +# A CVM registered on one node must appear on the other, and each node's two +# views of it -- the replicated store and the proxy's own state -- must agree. +test_cross_node_data_sync() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 5 + + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + local client_ip + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" app1 inst1)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + sleep 20 + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + return $ok +} + +# An opportunistic push must land well inside the 5s periodic interval. +test_push_fast_path() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + local before + before=$(get_n_instances 2) + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" push_app push_instance)" \ + >/dev/null || return 1 + + # 3s < the 5s periodic interval, so arriving in time is what proves the push + # path ran rather than the anti-entropy round that would have caught it anyway. + wait_for_instances 2 $((before + 1)) 3 || { + log_error "node 2 did not receive the write before the periodic interval"; return 1; } + wait_for_digest_match persistent 1 2 3 || { + log_error "persistent digests did not converge after push"; return 1; } +} + +# A write made while a peer was down must still reach it, via the periodic round. +test_periodic_repair_after_missed_push() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + local before + before=$(get_n_instances 2) + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" repair_app repair_instance)" \ + >/dev/null || return 1 + sleep 1 + start_node 2 || return 1 + setup_peers 1 2 + + wait_for_instances 2 $((before + 1)) 15 || { + log_error "periodic sync did not repair the missed write"; return 1; } +} + +# Losing the local store must not cost the node its identity, and it must +# refill from the peer rather than come back empty. +test_bootstrap_after_data_dir_loss() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" bootstrap_app bootstrap_instance)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || return 1 + + local old_uuid new_uuid + old_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ "$old_uuid" != "null" ]; } || { + log_error "node 2 did not report its identity before recovery"; return 1; } + + stop_node 2 + wipe_data_keeping_uuid 2 + start_node 2 || return 1 + + wait_for_instances 2 1 15 || { + log_error "node 2 did not bootstrap after losing its local store"; return 1; } + # get_store_digest ends in `|| true`, so an unreachable node yields an empty + # string and empty equals empty. wait_for_instances above speaks for node 2; + # nothing has spoken for node 1, so check both are non-empty before + # concluding anything from their being equal. + local d1 d2 + d1=$(get_store_digest 1 persistent); d2=$(get_store_digest 2 persistent) + { [ -n "$d1" ] && [ -n "$d2" ]; } || { + log_error "could not read both digests, so comparing them proves nothing: node1='${d1}' node2='${d2}'" + return 1; } + [ "$d1" = "$d2" ] || { + log_error "persistent digests differ after bootstrap: node1='${d1}' node2='${d2}'"; return 1; } + + setup_peers 1 2 + sleep 6 + new_uuid=$(get_node_uuid 2) + { [ -n "$new_uuid" ] && [ "$new_uuid" != "null" ]; } || { + log_error "node 2 did not report its post-recovery identity"; return 1; } + [ "$old_uuid" = "$new_uuid" ] || { + log_error "losing the WaveKV store unexpectedly changed the node UUID"; return 1; } +} + +# Writes made on both sides of a partition must both survive the merge. +test_divergent_partition_writes() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + stop_node 2 + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" left_app left_instance)" \ + >/dev/null || return 1 + + stop_node 1 + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA=" right_app right_instance)" \ + >/dev/null || return 1 + + start_node 1 || return 1 + setup_peers 1 2 + { wait_for_instances 1 2 15 && wait_for_instances 2 2 15; } || { + log_error "divergent partition writes did not merge"; return 1; } + wait_for_digest_match persistent 1 2 10 || { + log_error "persistent digests did not converge after divergent writes"; return 1; } +} + +# Writes that land while a periodic round is in flight must neither be lost nor +# counted twice. +test_push_periodic_overlap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 4 + + local before i + before=$(get_n_instances 1) + for i in $(seq 1 6); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((100 + i)))" "overlap_app_$i" "overlap_instance_$i")" \ + >/dev/null || { log_error "overlap write $i was rejected"; return 1; } + sleep 0.2 + done + + wait_for_instances 2 $((before + 6)) 15 || { + log_error "writes racing periodic sync did not arrive"; return 1; } + [ "$(get_n_instances 1)" -eq $((before + 6)) ] || { + log_error "overlapping push and sync produced duplicate instances"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# A node whose bootnode is not up yet must keep retrying and still converge. +test_delayed_bootnode_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + start_node 2 || return 1 + verify_register_response \ + "$(debug_register_cvm 2 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" delayed_app delayed_instance)" \ + >/dev/null || return 1 + + sleep 2 + start_node 1 || return 1 + + local _ + for _ in $(seq 1 25); do + has_peer_addr 1 2 && has_peer_addr 2 1 && break + sleep 1 + done + { has_peer_addr 1 2 && has_peer_addr 2 1; } || { + log_error "bootnode retry did not form the cluster"; return 1; } + wait_for_instances 1 1 15 || { + log_error "data did not converge after delayed bootnode recovery"; return 1; } +} + +# Killing a node mid-sync must not wedge it; the next round has to finish the job. +test_interrupted_sync_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + stop_node 2 + local i + for i in $(seq 1 20); do + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key $((200 + i)))" "interrupt_app_$i" "interrupt_instance_$i")" \ + >/dev/null || { log_error "interrupted-sync fixture write $i was rejected"; return 1; } + done + + # Bring it up, let a sync begin, then cut it off again. The exact instant + # does not matter -- what is asserted is that the following round still + # converges, not that the cut landed at a particular byte. + start_node 2 || return 1 + setup_peers 1 2 + sleep 0.2 + stop_node 2 + + start_node 2 || return 1 + setup_peers 1 2 + wait_for_instances 2 20 20 || { + log_error "sync did not recover after interruption"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# The ephemeral store is not journalled, so after a restart it has to be rebuilt +# from peers rather than read back from disk. +test_ephemeral_recovery() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 8 + + stop_node 2 + sleep 2 + start_node 2 || return 1 + setup_peers 1 2 + + local keys1 keys2 _ + for _ in $(seq 1 20); do + keys1=$(debug_get_sync_data 1 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + keys2=$(debug_get_sync_data 2 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])" 2>/dev/null || echo 0) + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } && break + sleep 1 + done + + # Assert what the loop above only waited for. Matching digests are not + # enough on their own: two nodes that never found each other both hold an + # empty ephemeral store, and empty matches empty. Stubbing out setup_peers + # left this test green, which is how the gap was found. + { [ "${keys1:-0}" -gt 0 ] && [ "${keys2:-0}" -gt 0 ]; } || { + log_error "ephemeral stores are empty, so a digest match proves nothing: keys1=${keys1:-0} keys2=${keys2:-0}" + return 1; } + + wait_for_digest_match ephemeral 1 2 15 || { + log_error "ephemeral store did not converge after restart"; return 1; } +} + +# A joining node must be able to bootstrap from the one peer that is up, even +# though another cluster member is unreachable. +test_partial_cluster_bootstrap() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 6 + + verify_register_response \ + "$(debug_register_cvm 1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" partial_app partial_instance)" \ + >/dev/null || { log_error "node 1 rejected the registration"; return 1; } + # 15s, not 10: every comparable wait in this suite allows 15, and this one + # is a peer sync over a link that setup_peers has only just established. + wait_for_instances 2 1 15 || { + log_error "node 2 never saw the instance registered on node 1"; return 1; } + + stop_node 2 + start_node 3 || return 1 + wait_for_instances 3 1 20 || { + log_error "node 3 did not bootstrap while node 2 was unavailable"; return 1; } +} + +# A node that lost its identity but kept its node id must be rejected on the +# first exchange, must not cost the established peer any data, and must then be +# allowed to rejoin under its new identity rather than staying wedged. +test_node_id_reuse_rejected() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 10 + + has_peer_addr 1 2 || { log_error "node 1 missing peer_addr for node 2"; return 1; } + has_peer_addr 2 1 || { log_error "node 2 missing peer_addr for node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 400)" reuse_app reuse_fixture)" \ + >/dev/null || return 1 + wait_for_instances 2 1 10 || { log_error "node 2 did not receive the recovery fixture"; return 1; } + + local old_uuid new_uuid keys_before keys_after + old_uuid=$(get_node_uuid 2) + keys_before=$(get_n_keys 1) + + + stop_node 2 + wipe_data 2 + start_node 2 || return 1 + + new_uuid=$(get_node_uuid 2) + { [ -n "$old_uuid" ] && [ -n "$new_uuid" ] && [ "$old_uuid" != "$new_uuid" ]; } || { + log_error "fresh node 2 did not receive a new UUID"; return 1; } + + setup_peers 1 2 + + local seen=false log1 log2 _ + for _ in $(seq 1 15); do + log1=$(dump_log 1); log2=$(dump_log 2) + if grep -q "UUID mismatch" "$log1" "$log2" 2>/dev/null; then seen=true; break; fi + sleep 1 + done + [ "$seen" = true ] || { log_error "reused node ID was not rejected"; return 1; } + + # Deliberate deviation from the suite this was ported from, which compared + # total key counts before and after and required the count not to drop. + # + # It does drop, by exactly one, and permanently: node 1 discards node 2's + # superseded identity record, which is the correct thing to do with it. + # Measured 9 -> 8, still 8 thirty seconds later, with the registered + # instance present throughout. So the old assertion forbids a legitimate + # deletion, and passed only when the replacement record happened to land + # before the count was taken. + # + # What the test is actually about -- its own comment says "node 1's data is + # still intact" -- is the replicated instance records, so assert on those. + local instances_after + instances_after=$(get_n_instances 1) + [ "$instances_after" -ge 1 ] 2>/dev/null || { + log_error "node 1 lost its registered instances after node 2 restarted with a reused ID: $instances_after" + return 1; } + log_info "node 1 keys ${keys_before} -> $(get_n_keys 1), instances ${instances_after}" + + wait_for_instances 2 1 20 || { + log_error "fresh node did not recover after the UUID rejection"; return 1; } + wait_for_digest_match persistent 1 2 15 || { + log_error "stores did not converge after UUID recovery"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 2 "$(test_public_key 401)" reuse_app post_recovery)" \ + >/dev/null || return 1 + wait_for_instances 1 2 15 || { log_error "post-recovery write did not propagate"; return 1; } + wait_for_digest_match persistent 1 2 10 +} + +# --------------------------------------------------------------- advanced + +# A registration must survive a restart, and the store must hold more than the +# handful of keys a bare node writes for itself. +test_client_registration_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local client_ip keys_before keys_after + client_ip=$(verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 502)" persist_app persist_inst)") + [ -n "$client_ip" ] || { log_error "registration failed"; return 1; } + + keys_before=$(get_n_keys 1) + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + + { [ "$keys_after" -ge "$keys_before" ] && [ "$keys_before" -gt 2 ]; } || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +test_stress_writes() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key app inst success=0 + for i in $(seq 1 10); do + key=$(test_public_key $((600 + i))) + app=$(printf "stressapp%02d" "$i") + inst=$(printf "stressinst%02d" "$i") + verify_register_response "$(debug_register_cvm 1 "$key" "$app" "$inst")" >/dev/null 2>&1 \ + && success=$((success + 1)) + done + sleep 2 + + local keys_after + keys_after=$(get_n_keys 1) + { [ "$success" -eq 10 ] && [ "$keys_after" -gt 2 ]; } || { + log_error "success=$success keys_after=$keys_after"; return 1; } +} + +# Writes accepted while the peer was gone must reach it afterwards, and each +# node's two views must still agree once they do. +test_network_partition() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 5 + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + stop_node 2 + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((700 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "partition_app$i" "partition_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + + local kv1_during ps1_during + kv1_during=$(get_n_instances 1) + ps1_during=$(get_n_proxy_state_instances 1) + + start_node 2 || return 1 + setup_peers 1 2 + sleep 15 + + local kv1 kv2 ps1 ps2 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + + { [ "$success" -eq 3 ] && [ "$kv1_during" -ge 3 ]; } || { + log_error "registration or KvStore write failed during partition"; ok=1; } + [ "$kv2" -ge "$kv1_during" ] || { + log_error "node 2 KvStore sync failed: kv2=$kv2 expected >= $kv1_during"; ok=1; } + [ "$ps2" -ge "$kv1_during" ] || { + log_error "node 2 ProxyState sync failed: ps2=$ps2 expected >= $kv1_during"; ok=1; } + [ "$kv1" -eq "$ps1" ] || { log_error "node 1 inconsistent: KvStore=$kv1 ProxyState=$ps1"; ok=1; } + [ "$kv2" -eq "$ps2" ] || { log_error "node 2 inconsistent: KvStore=$kv2 ProxyState=$ps2"; ok=1; } + log_info "ps1_during=$ps1_during" + return $ok +} + +_three_node_views_agree() { + local kv1 kv2 kv3 ps1 ps2 ps3 ok=0 + kv1=$(get_n_instances 1); kv2=$(get_n_instances 2); kv3=$(get_n_instances 3) + ps1=$(get_n_proxy_state_instances 1); ps2=$(get_n_proxy_state_instances 2) + ps3=$(get_n_proxy_state_instances 3) + { [ "$kv1" -ge 1 ] && [ "$kv2" -ge 1 ] && [ "$kv3" -ge 1 ]; } || { + log_error "KvStore sync failed: kv1=$kv1 kv2=$kv2 kv3=$kv3"; ok=1; } + { [ "$ps1" -ge 1 ] && [ "$ps2" -ge 1 ] && [ "$ps3" -ge 1 ]; } || { + log_error "ProxyState sync failed: ps1=$ps1 ps2=$ps2 ps3=$ps3"; ok=1; } + { [ "$kv1" -eq "$ps1" ] && [ "$kv2" -eq "$ps2" ] && [ "$kv3" -eq "$ps3" ]; } || { + log_error "inconsistency between KvStore and ProxyState"; ok=1; } + return $ok +} + +test_three_node_cluster() { + cleanup_cluster + generate_config 1 ""; generate_config 2 ""; generate_config 3 "" + start_node 1 || return 1 + start_node 2 || return 1 + start_node 3 || return 1 + setup_peers 1 2 3 + sleep 10 + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 503)" threenode_app threenode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + sleep 20 + _three_node_views_agree +} + +# The same shape, but the two joiners discover the cluster through a bootnode +# instead of being told about each other. +test_three_node_bootnode() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "https://gateway-1:9012" + generate_config 3 "https://gateway-1:9012" + start_node 1 || return 1 + sleep 2 + start_node 2 || return 1 + start_node 3 || return 1 + sleep 15 + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + verify_register_response \ + "$(debug_register_cvm 1 "$(test_public_key 504)" bootnode_app bootnode_inst)" \ + >/dev/null || { log_error "registration failed"; return 1; } + sleep 20 + _three_node_views_agree +} + +# The periodic persist must actually run -- and be seen to run -- and what it +# wrote must come back after a restart. +test_periodic_persistence() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local i key success=0 + for i in $(seq 1 3); do + key=$(test_public_key $((900 + i))) + verify_register_response "$(debug_register_cvm 1 "$key" "persist_app$i" "persist_inst$i")" \ + >/dev/null 2>&1 && success=$((success + 1)) + done + [ "$success" -eq 3 ] || { log_error "registered only $success/3 clients"; return 1; } + + local keys_before keys_after log wal_size + keys_before=$(get_n_keys 1) + sleep 8 + + log=$(dump_log 1) + grep -q "periodic persist completed" "$log" || { + log_error "periodic persist message not found in the log"; return 1; } + + wal_size=$(data_op "stat -c%s /data/node1/wavekv/node_1.wal 2>/dev/null || echo 0" | tr -d '\r') + [ "${wal_size:-0}" -gt 0 ] 2>/dev/null || { log_error "WAL file missing or empty"; return 1; } + log_info "WAL size after periodic persist: ${wal_size} bytes" + + stop_node 1 + start_node 1 || return 1 + keys_after=$(get_n_keys 1) + [ "$keys_after" -ge "$keys_before" ] || { + log_error "keys_before=$keys_before keys_after=$keys_after"; return 1; } +} + +# ------------------------------------------------------------------ admin + +test_admin_set_node_url() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local new_url="https://new-node2.example.com:8011" response stored + response=$(admin_set_node_url 1 2 "$new_url") + echo "$response" | grep -q '"error"' && { log_error "SetNodeUrl returned: $response"; return 1; } + + sleep 2 + stored=$(get_peer_url_from_sync 1 2) + [ "$stored" = "$new_url" ] || { + log_error "expected '$new_url', got '$stored'"; return 1; } +} + +test_admin_set_node_status() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + admin_set_node_url 1 2 "https://node2.example.com:8011" >/dev/null + sleep 1 + + local response + for state in down up; do + response=$(admin_set_node_status 1 2 "$state") + # Absence of "error" is not enough on its own: a request that never + # reached the node yields an empty string, which contains no "error" + # either, so a total connectivity failure would pass. + [ -n "$response" ] || { + log_error "SetNodeStatus($state) returned nothing at all"; return 1; } + echo "$response" | grep -q '"error"' && { + log_error "SetNodeStatus($state) returned: $response"; return 1; } + sleep 1 + done + + # Kept as a warning, exactly as the original had it: whether an unknown + # status is refused is not what this test is here to pin down. + response=$(admin_set_node_status 1 2 invalid) + echo "$response" | grep -q '"error"' || \ + log_warn "invalid status was not rejected (may be acceptable)" + return 0 +} + +# A node marked down must not be handed to registering CVMs, and must be handed +# out again once it is back up. +test_node_status_register_exclude() { + cleanup_cluster + generate_config 1 "" + generate_config 2 "" + start_node 1 || return 1 + start_node 2 || return 1 + setup_peers 1 2 + sleep 5 + check_debug_service 1 || { log_error "debug service not available on node 1"; return 1; } + + admin_set_node_status 1 2 down >/dev/null + sleep 2 + local response + response=$(debug_register_cvm 1 "$(test_public_key 505)" downtest_app downtest_inst) + verify_register_response "$response" >/dev/null || { log_error "registration failed"; return 1; } + response_lists_gateway "$response" 2 && { + log_error "node 2 (down) was included in the registration response"; return 1; } + + admin_set_node_status 1 2 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 506)" uptest_app uptest_inst2) + response_lists_gateway "$response" 2 || { + log_error "node 2 (up) was not included in the registration response"; return 1; } +} + +# A node marked down must refuse to register CVMs itself. +test_node_status_register_reject() { + cleanup_cluster + generate_config 1 "" + start_node 1 || return 1 + check_debug_service 1 || { log_error "debug service not available"; return 1; } + + local response + response=$(debug_register_cvm 1 "$(test_public_key 507)" upnode_app upnode_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed while the node was up"; return 1; } + + admin_set_node_status 1 1 down >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 508)" downnode_app downnode_inst) + echo "$response" | grep -qi "error" || { + log_error "registration was not rejected while the node is down"; return 1; } + + admin_set_node_status 1 1 up >/dev/null + sleep 2 + response=$(debug_register_cvm 1 "$(test_public_key 509)" backup_app backup_inst) + verify_register_response "$response" >/dev/null || { + log_error "registration failed once the node was back up"; return 1; } +} diff --git a/dstack/gateway/test-run/test_suite.sh b/dstack/gateway/test-run/test_suite.sh deleted file mode 100755 index 47570c4a7..000000000 --- a/dstack/gateway/test-run/test_suite.sh +++ /dev/null @@ -1,2447 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# WaveKV integration test script -# -# This legacy integration script predates the repository-wide shellcheck hook. -# Keep the existing style warnings suppressed so small harness fixes do not -# require a full script rewrite. -# shellcheck disable=SC2015,SC2034,SC2086,SC2155,SC2164 - -# Don't use set -e as it causes issues with cleanup and test flow -# set -e - -# Disable job control messages (prevents "Killed" messages from messing up output) -set +m - -# Fix terminal output - ensure proper line endings -stty -echoctl 2>/dev/null || true - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -GATEWAY_BIN="${GATEWAY_BIN:-${SCRIPT_DIR}/../../target/release/dstack-gateway}" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -CURRENT_TEST="" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -cleanup() { - log_info "Cleaning up..." - # Kill only dstack-gateway processes started by this test (matching our specific config path) - # Use absolute path to avoid killing system dstack-gateway processes - pkill -9 -f "dstack-gateway -c ${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - sleep 1 - # Only delete WireGuard interfaces with sudo (these are our test interfaces) - sudo ip link delete wavekv-test1 2>/dev/null || true - sudo ip link delete wavekv-test2 2>/dev/null || true - sudo ip link delete wavekv-test3 2>/dev/null || true - # Clean up all wavekv data directories to prevent peer list contamination - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" 2>/dev/null || true - rm -f "$RUN_DIR/gateway-state-node"*.json 2>/dev/null || true - sleep 1 - stty sane 2>/dev/null || true -} - -trap cleanup EXIT - -# Generate node configs -# Usage: generate_config [bootnode_url] -generate_config() { - local node_id=$1 - local bootnode_url=${2:-""} - local rpc_port=$((13000 + node_id * 10 + 2)) - local wg_port=$((13000 + node_id * 10 + 3)) - local proxy_port=$((13000 + node_id * 10 + 4)) - local debug_port=$((13000 + node_id * 10 + 5)) - local admin_port=$((13000 + node_id * 10 + 6)) - local wg_ip="10.0.3${node_id}.1/24" - - # Use absolute paths to avoid Rocket's relative path resolution issues - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - cat >"$RUN_DIR/node${node_id}.toml" </dev/null | grep -q ":${port} "; then - return 0 - fi - sleep 1 - ((waited++)) - done - return 1 -} - -ensure_wg_interface() { - local node_id=$1 - local iface="wavekv-test${node_id}" - - # Check if interface exists, create if not - if ! ip link show "$iface" >/dev/null 2>&1; then - log_info "Creating WireGuard interface ${iface}..." - sudo ip link add "$iface" type wireguard || { - log_error "Failed to create WireGuard interface ${iface}" - return 1 - } - fi - return 0 -} - -start_node() { - local node_id=$1 - local config="${SCRIPT_DIR}/${RUN_DIR}/node${node_id}.toml" - local log_file="${LOG_DIR}/${CURRENT_TEST}_node${node_id}.log" - - # Calculate ports for this node - local admin_port=$((13000 + node_id * 10 + 6)) - local rpc_port=$((13000 + node_id * 10 + 2)) - - log_info "Starting node ${node_id}..." - - # Kill any existing test process for this node first (use absolute path to be precise) - pkill -9 -f "dstack-gateway -c ${config}" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${config}" >/dev/null 2>&1 || true - sleep 1 - - # Wait for ports to be free - if ! wait_for_port_free $admin_port; then - log_error "Port $admin_port still in use after waiting" - netstat -tlnp 2>/dev/null | grep ":${admin_port} " || true - return 1 - fi - if ! wait_for_port_free $rpc_port; then - log_error "Port $rpc_port still in use after waiting" - netstat -tlnp 2>/dev/null | grep ":${rpc_port} " || true - return 1 - fi - - # Ensure WireGuard interface exists before starting - if ! ensure_wg_interface "$node_id"; then - return 1 - fi - - mkdir -p "$RUN_DIR/wavekv_node${node_id}" - mkdir -p "$LOG_DIR" - (RUST_LOG=info "$GATEWAY_BIN" -c "$config" >"$log_file" 2>&1 &) - sleep 2 - - if pgrep -f "dstack-gateway.*${config}" >/dev/null; then - log_info "Node ${node_id} started successfully" - return 0 - else - log_error "Node ${node_id} failed to start" - cat "$log_file" - return 1 - fi -} - -stop_node() { - local node_id=$1 - local config="${SCRIPT_DIR}/${RUN_DIR}/node${node_id}.toml" - local admin_port=$((13000 + node_id * 10 + 6)) - - log_info "Stopping node ${node_id}..." - # Kill only the specific test process using absolute config path - pkill -9 -f "dstack-gateway -c ${config}" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${config}" >/dev/null 2>&1 || true - sleep 1 - - # Verify the port is free, otherwise force kill by PID - if ! wait_for_port_free $admin_port; then - log_warn "Node ${node_id} port still in use, forcing cleanup..." - # Find and kill the process holding the port - local pid=$(netstat -tlnp 2>/dev/null | grep ":${admin_port} " | awk '{print $7}' | cut -d'/' -f1) - if [[ -n "$pid" ]]; then - kill -9 "$pid" 2>/dev/null || true - sleep 1 - fi - fi - - # Reset terminal to fix any broken line endings - stty sane 2>/dev/null || true -} - -# Get WaveKV status via Admin.WaveKvStatus RPC -# Usage: get_status -get_status() { - local admin_port=$1 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.WaveKvStatus" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -get_n_keys() { - local admin_port=$1 - get_status "$admin_port" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['persistent']['n_keys'])" 2>/dev/null || echo "0" -} - -# Register CVM via debug port (no attestation required) -# Usage: debug_register_cvm -# Returns: JSON response -debug_register_cvm() { - local debug_port=$1 - local public_key=$2 - local app_id=${3:-"testapp"} - local instance_id=${4:-"testinstance"} - curl -s \ - -X POST "http://localhost:${debug_port}/prpc/RegisterCvm" \ - -H "Content-Type: application/json" \ - -d "{\"client_public_key\": \"$public_key\", \"app_id\": \"$app_id\", \"instance_id\": \"$instance_id\"}" 2>/dev/null -} - -# Check if debug service is available -# Usage: check_debug_service -check_debug_service() { - local debug_port=$1 - local response=$(curl -s -X POST "http://localhost:${debug_port}/prpc/Debug.Info" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null) - if echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); assert 'base_domain' in d" 2>/dev/null; then - return 0 - else - return 1 - fi -} - -# Verify register response is successful (has wg config, no error) -# Usage: verify_register_response -verify_register_response() { - local response="$1" - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - if 'error' in d: - print(f'ERROR: {d[\"error\"]}', file=sys.stderr) - sys.exit(1) - assert 'wg' in d, 'missing wg config' - assert 'client_ip' in d['wg'], 'missing client_ip' - print(d['wg']['client_ip']) -except Exception as e: - print(f'ERROR: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null -} - -# Get sync data from debug port (peer_addrs, nodes, instances) -# Usage: debug_get_sync_data -# Returns: JSON response with my_node_id, peer_addrs, nodes, instances -debug_get_sync_data() { - local debug_port=$1 - curl -s -X POST "http://localhost:${debug_port}/prpc/Debug.GetSyncData" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null -} - -# Check if node has synced peer address from another node -# Usage: has_peer_addr -# Returns: 0 if peer address exists, 1 otherwise -has_peer_addr() { - local debug_port=$1 - local peer_node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - peer_addrs = d.get('peer_addrs', []) - for pa in peer_addrs: - if pa.get('node_id') == $peer_node_id: - sys.exit(0) - sys.exit(1) -except Exception as e: - sys.exit(1) -" -} - -# Check if node has synced node info from another node -# Usage: has_node_info -# Returns: 0 if node info exists, 1 otherwise -has_node_info() { - local debug_port=$1 - local peer_node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - nodes = d.get('nodes', []) - for n in nodes: - if n.get('node_id') == $peer_node_id: - sys.exit(0) - sys.exit(1) -except Exception as e: - sys.exit(1) -" -} - -# Get number of peer addresses from sync data -# Usage: get_n_peer_addrs -get_n_peer_addrs() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('peer_addrs', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Get number of node infos from sync data -# Usage: get_n_nodes -get_n_nodes() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('nodes', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Get number of instances from KvStore sync data -# Usage: get_n_instances -get_n_instances() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('instances', []))) -except Exception: - print(0) -" 2>/dev/null -} - -get_persistent_digest() { - local admin_port=$1 - get_status "$admin_port" | python3 -c "import sys,json; print(json.load(sys.stdin)['persistent']['digest'])" 2>/dev/null || true -} - -get_ephemeral_digest() { - local admin_port=$1 - get_status "$admin_port" | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral']['digest'])" 2>/dev/null || true -} - -get_node_uuid() { - local admin_port=$1 - admin_get_status "$admin_port" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)['uuid']))" 2>/dev/null || true -} - -# A deterministic, *valid* WireGuard public key: 32 bytes, so 44 base64 -# characters. Registration rejects anything else (see WG_PUBLIC_KEY_B64_LEN in -# gateway/src/kv/import.rs), so a hand-written placeholder is not a shortcut — -# it is a test that can only fail. Use a distinct seed per registration; two -# instances sharing a public key are refused as a conflict. -test_public_key() { - local seed=$1 - python3 -c "import base64; print(base64.b64encode(int($seed).to_bytes(32, 'big')).decode())" -} - -wait_for_instances() { - local debug_port=$1 - local expected=$2 - local timeout_seconds=$3 - local attempts=$((timeout_seconds * 10)) - for _ in $(seq 1 "$attempts"); do - if [[ "$(get_n_instances "$debug_port")" -ge "$expected" ]]; then return 0; fi - sleep 0.1 - done - return 1 -} - -wait_for_digest_match() { - local store=$1 - local port1=$2 - local port2=$3 - local timeout_seconds=$4 - local getter="get_${store}_digest" - for _ in $(seq 1 $((timeout_seconds * 10))); do - local digest1=$($getter "$port1") - local digest2=$($getter "$port2") - if [[ -n "$digest1" && "$digest1" == "$digest2" ]]; then return 0; fi - sleep 0.1 - done - return 1 -} - -# Get Proxy State from debug port (in-memory state) -# Usage: debug_get_proxy_state -# Returns: JSON response with instances and allocated_addresses -debug_get_proxy_state() { - local debug_port=$1 - curl -s -X POST "http://localhost:${debug_port}/prpc/GetProxyState" \ - -H "Content-Type: application/json" -d '{}' 2>/dev/null -} - -# Get number of instances from ProxyState (in-memory) -# Usage: get_n_proxy_state_instances -get_n_proxy_state_instances() { - local debug_port=$1 - local response=$(debug_get_proxy_state "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - print(len(d.get('instances', []))) -except Exception: - print(0) -" 2>/dev/null -} - -# Check KvStore and ProxyState instance consistency -# Usage: check_instance_consistency -# Returns: 0 if consistent, 1 otherwise -check_instance_consistency() { - local debug_port=$1 - local kvstore_instances=$(get_n_instances "$debug_port") - local proxystate_instances=$(get_n_proxy_state_instances "$debug_port") - - if [[ "$kvstore_instances" -eq "$proxystate_instances" ]]; then - return 0 - else - log_error "Instance count mismatch: KvStore=$kvstore_instances, ProxyState=$proxystate_instances" - return 1 - fi -} - -# ============================================================================= -# Test 1: Single node persistence -# ============================================================================= -test_persistence() { - log_info "========== Test 1: Persistence ==========" - cleanup - - generate_config 1 - - # Start node and let it write some data - start_node 1 - - local admin_port=13016 - local initial_keys=$(get_n_keys $admin_port) - log_info "Initial keys: $initial_keys" - - # The gateway auto-writes some data (peer_addr, etc) - sleep 2 - local keys_after_write=$(get_n_keys $admin_port) - log_info "Keys after startup: $keys_after_write" - - # Stop and restart - stop_node 1 - log_info "Restarting node 1..." - start_node 1 - - local keys_after_restart=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after_restart" - - if [[ "$keys_after_restart" -ge "$keys_after_write" ]]; then - log_info "Persistence test PASSED" - return 0 - else - log_error "Persistence test FAILED: expected >= $keys_after_write keys, got $keys_after_restart" - return 1 - fi -} - -# ============================================================================= -# Test 2: Multi-node sync -# ============================================================================= -test_multi_node_sync() { - log_info "========== Test 2: Multi-node Sync ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for sync - log_info "Waiting for nodes to sync..." - sleep 10 - - # Use debug RPC to check actual synced data - local peer_addrs1=$(get_n_peer_addrs $debug_port1) - local peer_addrs2=$(get_n_peer_addrs $debug_port2) - local nodes1=$(get_n_nodes $debug_port1) - local nodes2=$(get_n_nodes $debug_port2) - - log_info "Node 1: peer_addrs=$peer_addrs1, nodes=$nodes1" - log_info "Node 2: peer_addrs=$peer_addrs2, nodes=$nodes2" - - # For true sync, each node should have: - # - At least 2 peer addresses (both nodes' addresses) - # - At least 2 node infos (both nodes' info) - local sync_ok=true - - if ! has_peer_addr $debug_port1 2; then - log_error "Node 1 missing peer_addr for node 2" - sync_ok=false - fi - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1" - sync_ok=false - fi - if ! has_node_info $debug_port1 2; then - log_error "Node 1 missing node_info for node 2" - sync_ok=false - fi - if ! has_node_info $debug_port2 1; then - log_error "Node 2 missing node_info for node 1" - sync_ok=false - fi - - if [[ "$sync_ok" == "true" ]]; then - log_info "Multi-node sync test PASSED" - return 0 - else - log_error "Multi-node sync test FAILED: nodes did not sync peer data" - log_info "Sync data from node 1: $(debug_get_sync_data $debug_port1)" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 3: Node recovery after disconnect -# ============================================================================= -test_node_recovery() { - log_info "========== Test 3: Node Recovery ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for initial sync - sleep 5 - - # Stop node 2 - log_info "Stopping node 2 to simulate disconnect..." - stop_node 2 - - # Wait and let node 1 continue - sleep 3 - - # Check node 1 has its own data - local peer_addrs1_before=$(get_n_peer_addrs $debug_port1) - log_info "Node 1 peer_addrs before node 2 restart: $peer_addrs1_before" - - # Restart node 2 - log_info "Restarting node 2..." - start_node 2 - - # Re-register peers after restart - setup_peers 1 2 - - # Wait for sync - sleep 10 - - # After recovery, node 2 should have synced node 1's data - local sync_ok=true - - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1 after recovery" - sync_ok=false - fi - if ! has_node_info $debug_port2 1; then - log_error "Node 2 missing node_info for node 1 after recovery" - sync_ok=false - fi - - if [[ "$sync_ok" == "true" ]]; then - log_info "Node recovery test PASSED" - return 0 - else - log_error "Node recovery test FAILED: node 2 did not sync data from node 1" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 4: Status endpoint structure (Admin.WaveKvStatus RPC) -# ============================================================================= -test_status_endpoint() { - log_info "========== Test 4: Status Endpoint ==========" - cleanup - - generate_config 1 - start_node 1 - - local admin_port=13016 - local status=$(get_status $admin_port) - - # Verify all expected fields exist - local checks_passed=0 - local total_checks=6 - - echo "$status" | python3 -c " -import sys, json -d = json.load(sys.stdin) -assert d['enabled'] == True, 'enabled should be True' -assert 'persistent' in d, 'missing persistent' -assert 'ephemeral' in d, 'missing ephemeral' -assert d['persistent']['wal_enabled'] == True, 'persistent wal should be enabled' -assert d['ephemeral']['wal_enabled'] == False, 'ephemeral wal should be disabled' -assert 'peers' in d['persistent'], 'missing peers in persistent' -print('All status checks passed') -" && checks_passed=1 - - if [[ $checks_passed -eq 1 ]]; then - log_info "Status endpoint test PASSED" - return 0 - else - log_error "Status endpoint test FAILED" - log_info "Status response: $status" - return 1 - fi -} - -# ============================================================================= -# Test 5: Cross-node data sync verification (KvStore + ProxyState) -# ============================================================================= -test_cross_node_data_sync() { - log_info "========== Test 5: Cross-node Data Sync ==========" - cleanup - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Wait for initial connection - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Register a client on node 1 via debug port - log_info "Registering client on node 1 via debug port..." - local register_response=$(debug_register_cvm $debug_port1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "app1" "inst1") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync (need at least 3 sync intervals of 5s for data to propagate) - log_info "Waiting for sync..." - sleep 20 - - # Check KvStore instance count on both nodes - local kv_instances1=$(get_n_instances $debug_port1) - local kv_instances2=$(get_n_instances $debug_port2) - - # Check ProxyState instance count on both nodes - local ps_instances1=$(get_n_proxy_state_instances $debug_port1) - local ps_instances2=$(get_n_proxy_state_instances $debug_port2) - - log_info "Node 1: KvStore=$kv_instances1, ProxyState=$ps_instances1" - log_info "Node 2: KvStore=$kv_instances2, ProxyState=$ps_instances2" - - local test_passed=true - - # Verify KvStore sync - if [[ "$kv_instances1" -lt 1 ]] || [[ "$kv_instances2" -lt 1 ]]; then - log_error "KvStore sync failed: kv_instances1=$kv_instances1, kv_instances2=$kv_instances2" - test_passed=false - fi - - # Verify ProxyState sync (node 2 should have loaded instance from KvStore) - if [[ "$ps_instances1" -lt 1 ]] || [[ "$ps_instances2" -lt 1 ]]; then - log_error "ProxyState sync failed: ps_instances1=$ps_instances1, ps_instances2=$ps_instances2" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv_instances1" -ne "$ps_instances1" ]]; then - log_error "Node 1 inconsistent: KvStore=$kv_instances1, ProxyState=$ps_instances1" - test_passed=false - fi - if [[ "$kv_instances2" -ne "$ps_instances2" ]]; then - log_error "Node 2 inconsistent: KvStore=$kv_instances2, ProxyState=$ps_instances2" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Cross-node data sync test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 1: $(debug_get_sync_data $debug_port1)" - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "ProxyState from node 1: $(debug_get_proxy_state $debug_port1)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Push fast path: propagation must happen before the periodic interval -# ============================================================================= -test_push_fast_path() { - log_info "========== Push Fast Path ==========" - cleanup - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - - local before=$(get_n_instances 13025) - local response=$(debug_register_cvm 13015 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "push_app" "push_instance") - verify_register_response "$response" >/dev/null || return 1 - # The periodic interval is 5 seconds. Arrival within 3 seconds exercises push. - wait_for_instances 13025 $((before + 1)) 3 || { - log_error "Node 2 did not receive the write before the periodic interval"; return 1; } - for _ in $(seq 1 30); do - local digest1=$(get_persistent_digest 13016) - local digest2=$(get_persistent_digest 13026) - if [[ -n "$digest1" && "$digest1" == "$digest2" ]]; then - log_info "Push fast-path and digest convergence test PASSED"; return 0 - fi - sleep 0.1 - done - log_error "Persistent digests did not converge after push" - return 1 -} - -# ============================================================================= -# Periodic anti-entropy must repair a write whose push could not be delivered -# ============================================================================= -test_periodic_repair_after_missed_push() { - log_info "========== Periodic Repair After Missed Push ==========" - cleanup - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - - local before=$(get_n_instances 13025) - stop_node 2 - local response=$(debug_register_cvm 13015 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "repair_app" "repair_instance") - verify_register_response "$response" >/dev/null || return 1 - sleep 1 - start_node 2; setup_peers 1 2 - wait_for_instances 13025 $((before + 1)) 15 || { - log_error "Periodic sync did not repair the missed write"; return 1; } - log_info "Periodic repair test PASSED" -} - -# ============================================================================= -# A node that loses its local store files must bootstrap before local writes -# ============================================================================= -test_bootstrap_after_data_dir_loss() { - log_info "========== Bootstrap After Data Directory Loss ==========" - cleanup - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - generate_config 1 - generate_config 2 "https://localhost:13012" - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - - local response=$(debug_register_cvm 13015 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "bootstrap_app" "bootstrap_instance") - verify_register_response "$response" >/dev/null || return 1 - wait_for_instances 13025 1 10 || return 1 - local old_uuid=$(get_node_uuid 13026) - if [[ -z "$old_uuid" || "$old_uuid" == "null" ]]; then - log_error "Node 2 did not report its identity before recovery"; return 1 - fi - - stop_node 2 - cp "$RUN_DIR/wavekv_node2/node_uuid" "$RUN_DIR/node2.uuid" - rm -rf "$RUN_DIR/wavekv_node2" - mkdir -p "$RUN_DIR/wavekv_node2" - mv "$RUN_DIR/node2.uuid" "$RUN_DIR/wavekv_node2/node_uuid" - start_node 2 - wait_for_instances 13025 1 15 || { - log_error "Node 2 did not bootstrap after losing its local store"; return 1; } - if [[ "$(get_persistent_digest 13016)" != "$(get_persistent_digest 13026)" ]]; then - log_error "Persistent digests differ after bootstrap"; return 1 - fi - setup_peers 1 2; sleep 6 - local new_uuid=$(get_node_uuid 13026) - if [[ -z "$new_uuid" || "$new_uuid" == "null" ]]; then - log_error "Node 2 did not report its post-recovery identity"; return 1 - fi - if [[ "$old_uuid" != "$new_uuid" ]]; then - log_error "Losing the WaveKV store unexpectedly changed the node UUID"; return 1 - fi - log_info "Data-directory-loss bootstrap test PASSED" -} - -# ============================================================================= -# Divergent writes made from the same base must merge after both nodes return -# ============================================================================= -test_divergent_partition_writes() { - log_info "========== Divergent Partition Writes ==========" - cleanup - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - - stop_node 2 - verify_register_response "$(debug_register_cvm 13015 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "left_app" "left_instance")" >/dev/null || return 1 - stop_node 1; start_node 2 - verify_register_response "$(debug_register_cvm 13025 \ - "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA=" "right_app" "right_instance")" >/dev/null || return 1 - start_node 1; setup_peers 1 2 - wait_for_instances 13015 2 15 && wait_for_instances 13025 2 15 || { - log_error "Divergent partition writes did not merge"; return 1; } - wait_for_digest_match persistent 13016 13026 10 || { - log_error "Persistent digests did not converge after divergent writes"; return 1; } - log_info "Divergent partition write test PASSED" -} - -# ============================================================================= -# Pushes racing the periodic round must remain idempotent and converge -# ============================================================================= -test_push_periodic_overlap() { - log_info "========== Push and Periodic Sync Overlap ==========" - cleanup - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 4 - local before=$(get_n_instances 13015) - for i in $(seq 1 6); do - verify_register_response "$(debug_register_cvm 13015 \ - "$(test_public_key $((100 + i)))" "overlap_app_$i" "overlap_instance_$i")" >/dev/null || { - log_error "Overlap write $i was rejected"; return 1; } - sleep 0.2 - done - wait_for_instances 13025 $((before + 6)) 15 || { - log_error "Writes racing periodic sync did not arrive"; return 1; } - [[ "$(get_n_instances 13015)" -eq $((before + 6)) ]] || { - log_error "Overlapping push and sync produced duplicate instances"; return 1; } - wait_for_digest_match persistent 13016 13026 10 || return 1 - log_info "Push/periodic overlap test PASSED" -} - -# ============================================================================= -# A node started before its bootnode must discover it on a later retry -# ============================================================================= -test_delayed_bootnode_recovery() { - log_info "========== Delayed Bootnode Recovery ==========" - cleanup - generate_config 1 - generate_config 2 "https://localhost:13012" - start_node 2 - verify_register_response "$(debug_register_cvm 13025 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "delayed_app" "delayed_instance")" >/dev/null || return 1 - sleep 2; start_node 1 - for _ in $(seq 1 25); do - has_peer_addr 13015 2 && has_peer_addr 13025 1 && break - sleep 1 - done - has_peer_addr 13015 2 && has_peer_addr 13025 1 || { - log_error "Bootnode retry did not form the cluster"; return 1; } - wait_for_instances 13015 1 15 || { - log_error "Data did not converge after delayed bootnode recovery"; return 1; } - log_info "Delayed bootnode recovery test PASSED" -} - -# ============================================================================= -# Interrupting a recovery round must not prevent a later round from converging -# ============================================================================= -test_interrupted_sync_recovery() { - log_info "========== Interrupted Sync Recovery ==========" - cleanup - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - stop_node 2 - for i in $(seq 1 20); do - verify_register_response "$(debug_register_cvm 13015 \ - "$(test_public_key $((200 + i)))" "interrupt_app_$i" "interrupt_instance_$i")" >/dev/null || { - log_error "Interrupted-sync fixture write $i was rejected"; return 1; } - done - start_node 2; setup_peers 1 2; sleep 0.2; stop_node 2 - start_node 2; setup_peers 1 2 - wait_for_instances 13025 20 20 || { - log_error "Sync did not recover after interruption"; return 1; } - wait_for_digest_match persistent 13016 13026 10 || return 1 - log_info "Interrupted sync recovery test PASSED" -} - -# ============================================================================= -# Ephemeral state must resume converging after a peer outage -# ============================================================================= -test_ephemeral_recovery() { - log_info "========== Ephemeral Store Recovery ==========" - cleanup - generate_config 1; generate_config 2 - start_node 1; start_node 2; setup_peers 1 2; sleep 8 - stop_node 2; sleep 2; start_node 2; setup_peers 1 2 - for _ in $(seq 1 20); do - local keys1=$(debug_get_sync_data 13015 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])") - local keys2=$(debug_get_sync_data 13025 | python3 -c "import sys,json; print(json.load(sys.stdin)['ephemeral_keys'])") - if [[ "$keys1" -gt 0 && "$keys2" -gt 0 ]]; then break; fi - sleep 1 - done - wait_for_digest_match ephemeral 13016 13026 15 || { - log_error "Ephemeral store did not converge after restart"; return 1; } - log_info "Ephemeral recovery test PASSED" -} - -# ============================================================================= -# A fresh third node must bootstrap while another non-bootnode peer is down -# ============================================================================= -test_partial_cluster_bootstrap() { - log_info "========== Partial Cluster Bootstrap ==========" - cleanup - generate_config 1; generate_config 2; generate_config 3 "https://localhost:13012" - start_node 1; start_node 2; setup_peers 1 2; sleep 6 - verify_register_response "$(debug_register_cvm 13015 \ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" "partial_app" "partial_instance")" >/dev/null || return 1 - wait_for_instances 13025 1 10 || return 1 - stop_node 2; start_node 3 - wait_for_instances 13035 1 20 || { - log_error "Node 3 did not bootstrap while node 2 was unavailable"; return 1; } - log_info "Partial-cluster bootstrap test PASSED" -} - -# ============================================================================= -# Test 6: prpc DebugRegisterCvm endpoint (on separate debug port) -# ============================================================================= -test_prpc_register() { - log_info "========== Test 6: prpc DebugRegisterCvm ==========" - cleanup - - generate_config 1 - start_node 1 - - local debug_port=13015 - - # Verify debug service is available first - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - log_info "Debug service is available" - - # Register via debug port - local register_response=$(debug_register_cvm $debug_port "$(test_public_key 501)" "deadbeef" "cafebabe") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "prpc DebugRegisterCvm test FAILED" - return 1 - fi - - log_info "DebugRegisterCvm success: client_ip=$client_ip" - log_info "prpc DebugRegisterCvm test PASSED" - return 0 -} - -# ============================================================================= -# Test 7: prpc Info endpoint -# ============================================================================= -test_prpc_info() { - log_info "========== Test 7: prpc Info ==========" - cleanup - - generate_config 1 - start_node 1 - - local port=13012 - - # Call Info via prpc - # Note: trim: "Tproxy." removes "Tproxy.Gateway." prefix, so endpoint is just /prpc/Info - local info_response=$(curl -sk --cacert "$CA_CERT" \ - -X POST "https://localhost:${port}/prpc/Info" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null) - - log_info "Info response: $info_response" - - # Verify response has expected fields and no error - echo "$info_response" | python3 -c " -import sys, json -d = json.load(sys.stdin) -if 'error' in d: - print(f'ERROR: {d[\"error\"]}', file=sys.stderr) - sys.exit(1) -assert 'base_domain' in d, 'missing base_domain' -assert 'external_port' in d, 'missing external_port' -print('prpc Info check passed') -" && { - log_info "prpc Info test PASSED" - return 0 - } || { - log_error "prpc Info test FAILED" - return 1 - } -} - -# ============================================================================= -# Test 8: Client registration and data persistence -# ============================================================================= -test_client_registration_persistence() { - log_info "========== Test 8: Client Registration Persistence ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register a client via debug port - log_info "Registering client..." - local register_response=$(debug_register_cvm $debug_port "$(test_public_key 502)" "persist_app" "persist_inst") - log_info "Register response: $register_response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$register_response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - - # Get initial key count - local keys_before=$(get_n_keys $admin_port) - log_info "Keys before restart: $keys_before" - - # Restart node - stop_node 1 - start_node 1 - - # Check keys after restart - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after" - - if [[ "$keys_after" -ge "$keys_before" ]] && [[ "$keys_before" -gt 2 ]]; then - log_info "Client registration persistence test PASSED" - return 0 - else - log_error "Client registration persistence test FAILED: keys_before=$keys_before, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Test 9: Stress test - multiple writes -# ============================================================================= -test_stress_writes() { - log_info "========== Test 9: Stress Test ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - local num_clients=10 - local success_count=0 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - log_info "Registering $num_clients clients via debug port..." - for i in $(seq 1 $num_clients); do - local key=$(test_public_key $((600 + i))) - local app_id=$(printf "stressapp%02d" "$i") - local inst_id=$(printf "stressinst%02d" "$i") - local response=$(debug_register_cvm $debug_port "$key" "$app_id" "$inst_id") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - - log_info "Successfully registered $success_count/$num_clients clients" - - sleep 2 - - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after stress test: $keys_after" - - # We expect successful registrations to create keys - if [[ "$success_count" -eq "$num_clients" ]] && [[ "$keys_after" -gt 2 ]]; then - log_info "Stress test PASSED" - return 0 - else - log_error "Stress test FAILED: success_count=$success_count, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Test 10: Network partition simulation (KvStore + ProxyState consistency) -# ============================================================================= -test_network_partition() { - log_info "========== Test 10: Network Partition Recovery ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - - # Let them sync initially - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Stop node 2 (simulate partition) - log_info "Simulating network partition - stopping node 2..." - stop_node 2 - - # Register clients on node 1 while node 2 is down - log_info "Registering clients on node 1 during partition..." - local success_count=0 - for i in $(seq 1 3); do - local key=$(test_public_key $((700 + i))) - local response=$(debug_register_cvm $debug_port1 "$key" "partition_app$i" "partition_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/3 clients during partition" - - local kv1_during=$(get_n_instances $debug_port1) - local ps1_during=$(get_n_proxy_state_instances $debug_port1) - log_info "Node 1 during partition: KvStore=$kv1_during, ProxyState=$ps1_during" - - # Restore node 2 - log_info "Healing partition - restarting node 2..." - start_node 2 - - # Re-register peers after restart - setup_peers 1 2 - - # Wait for sync - sleep 15 - - # Check KvStore and ProxyState on both nodes after recovery - local kv1_after=$(get_n_instances $debug_port1) - local kv2_after=$(get_n_instances $debug_port2) - local ps1_after=$(get_n_proxy_state_instances $debug_port1) - local ps2_after=$(get_n_proxy_state_instances $debug_port2) - - log_info "Node 1 after recovery: KvStore=$kv1_after, ProxyState=$ps1_after" - log_info "Node 2 after recovery: KvStore=$kv2_after, ProxyState=$ps2_after" - - local test_passed=true - - # Verify basic sync - if [[ "$success_count" -ne 3 ]] || [[ "$kv1_during" -lt 3 ]]; then - log_error "Registration or KvStore write failed during partition" - test_passed=false - fi - - # Verify node 2 synced KvStore - if [[ "$kv2_after" -lt "$kv1_during" ]]; then - log_error "Node 2 KvStore sync failed: kv2_after=$kv2_after, expected >= $kv1_during" - test_passed=false - fi - - # Verify node 2 ProxyState sync - if [[ "$ps2_after" -lt "$kv1_during" ]]; then - log_error "Node 2 ProxyState sync failed: ps2_after=$ps2_after, expected >= $kv1_during" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1_after" -ne "$ps1_after" ]]; then - log_error "Node 1 inconsistent: KvStore=$kv1_after, ProxyState=$ps1_after" - test_passed=false - fi - if [[ "$kv2_after" -ne "$ps2_after" ]]; then - log_error "Node 2 inconsistent: KvStore=$kv2_after, ProxyState=$ps2_after" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Network partition recovery test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - return 1 - fi -} - -# ============================================================================= -# Test 11: Three-node cluster (KvStore + ProxyState consistency) -# ============================================================================= -test_three_node_cluster() { - log_info "========== Test 11: Three-node Cluster ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - generate_config 1 - generate_config 2 - generate_config 3 - - start_node 1 - start_node 2 - start_node 3 - - # Register peers so all nodes can discover each other - setup_peers 1 2 3 - - local debug_port1=13015 - local debug_port2=13025 - local debug_port3=13035 - - # Wait for cluster to form - sleep 10 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Register client on node 1 - log_info "Registering client on node 1..." - local response=$(debug_register_cvm $debug_port1 "$(test_public_key 503)" "threenode_app" "threenode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync across all nodes (need at least 2 sync intervals of 5s) - sleep 20 - - # Check KvStore instances on all three nodes - local kv1=$(get_n_instances $debug_port1) - local kv2=$(get_n_instances $debug_port2) - local kv3=$(get_n_instances $debug_port3) - - # Check ProxyState instances on all three nodes - local ps1=$(get_n_proxy_state_instances $debug_port1) - local ps2=$(get_n_proxy_state_instances $debug_port2) - local ps3=$(get_n_proxy_state_instances $debug_port3) - - log_info "Node 1: KvStore=$kv1, ProxyState=$ps1" - log_info "Node 2: KvStore=$kv2, ProxyState=$ps2" - log_info "Node 3: KvStore=$kv3, ProxyState=$ps3" - - local test_passed=true - - # Verify KvStore sync on all nodes - if [[ "$kv1" -lt 1 ]] || [[ "$kv2" -lt 1 ]] || [[ "$kv3" -lt 1 ]]; then - log_error "KvStore sync failed: kv1=$kv1, kv2=$kv2, kv3=$kv3" - test_passed=false - fi - - # Verify ProxyState sync on all nodes - if [[ "$ps1" -lt 1 ]] || [[ "$ps2" -lt 1 ]] || [[ "$ps3" -lt 1 ]]; then - log_error "ProxyState sync failed: ps1=$ps1, ps2=$ps2, ps3=$ps3" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1" -ne "$ps1" ]] || [[ "$kv2" -ne "$ps2" ]] || [[ "$kv3" -ne "$ps3" ]]; then - log_error "Inconsistency detected between KvStore and ProxyState" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Three-node cluster test PASSED (KvStore and ProxyState consistent)" - return 0 - else - log_info "KvStore from node 1: $(debug_get_sync_data $debug_port1)" - log_info "KvStore from node 2: $(debug_get_sync_data $debug_port2)" - log_info "KvStore from node 3: $(debug_get_sync_data $debug_port3)" - log_info "ProxyState from node 1: $(debug_get_proxy_state $debug_port1)" - log_info "ProxyState from node 2: $(debug_get_proxy_state $debug_port2)" - log_info "ProxyState from node 3: $(debug_get_proxy_state $debug_port3)" - return 1 - fi -} - -# ============================================================================= -# Test 12: WAL file integrity -# ============================================================================= -test_wal_integrity() { - log_info "========== Test 12: WAL File Integrity ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local success_count=0 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register some clients via debug port - for i in $(seq 1 5); do - local key=$(test_public_key $((800 + i))) - local response=$(debug_register_cvm $debug_port "$key" "wal_app$i" "wal_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/5 clients" - - if [[ "$success_count" -ne 5 ]]; then - log_error "Failed to register all clients" - return 1 - fi - - sleep 2 - stop_node 1 - - # Check WAL file exists and has content - local wal_file="$RUN_DIR/wavekv_node1/node_1.wal" - if [[ -f "$wal_file" ]]; then - local wal_size=$(stat -c%s "$wal_file" 2>/dev/null || stat -f%z "$wal_file" 2>/dev/null) - log_info "WAL file size: $wal_size bytes" - - if [[ "$wal_size" -gt 100 ]]; then - log_info "WAL file integrity test PASSED" - return 0 - else - log_error "WAL file integrity test FAILED: WAL file too small ($wal_size bytes)" - return 1 - fi - else - log_error "WAL file not found: $wal_file" - return 1 - fi -} - -# ============================================================================= -# Test 13: Three-node cluster with bootnode (no dynamic peer setup) -# ============================================================================= -test_three_node_bootnode() { - log_info "========== Test 13: Three-node Cluster with Bootnode ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" "$RUN_DIR/wavekv_node3" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" "$RUN_DIR/gateway-state-node3.json" - - # Node 1 is the bootnode (no bootnode config) - # Node 2 and 3 use node 1 as bootnode - local bootnode_url="https://localhost:13012" - - generate_config 1 "" - generate_config 2 "$bootnode_url" - generate_config 3 "$bootnode_url" - - # Start node 1 first (bootnode) - start_node 1 - sleep 2 - - # Start node 2 and 3, they will discover each other via bootnode - start_node 2 - start_node 3 - - local debug_port1=13015 - local debug_port2=13025 - local debug_port3=13035 - - # Wait for cluster to form via bootnode discovery - log_info "Waiting for nodes to discover each other via bootnode..." - sleep 15 - - # Verify debug service is available on all nodes - for port in $debug_port1 $debug_port2 $debug_port3; do - if ! check_debug_service $port; then - log_error "Debug service not available on port $port" - return 1 - fi - done - - # Check peer discovery - each node should know about the others - local peer_addrs1=$(get_n_peer_addrs $debug_port1) - local peer_addrs2=$(get_n_peer_addrs $debug_port2) - local peer_addrs3=$(get_n_peer_addrs $debug_port3) - - log_info "Peer addresses: node1=$peer_addrs1, node2=$peer_addrs2, node3=$peer_addrs3" - - # Register client on node 2 (not the bootnode) - log_info "Registering client on node 2..." - local response=$(debug_register_cvm $debug_port2 "$(test_public_key 504)" "bootnode_app" "bootnode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Wait for sync across all nodes - sleep 20 - - # Check KvStore instances on all three nodes - local kv1=$(get_n_instances $debug_port1) - local kv2=$(get_n_instances $debug_port2) - local kv3=$(get_n_instances $debug_port3) - - # Check ProxyState instances on all three nodes - local ps1=$(get_n_proxy_state_instances $debug_port1) - local ps2=$(get_n_proxy_state_instances $debug_port2) - local ps3=$(get_n_proxy_state_instances $debug_port3) - - log_info "Node 1 (bootnode): KvStore=$kv1, ProxyState=$ps1" - log_info "Node 2: KvStore=$kv2, ProxyState=$ps2" - log_info "Node 3: KvStore=$kv3, ProxyState=$ps3" - - local test_passed=true - - # Verify peer discovery worked (each node should have at least 2 peer addresses) - if [[ "$peer_addrs1" -lt 2 ]] || [[ "$peer_addrs2" -lt 2 ]] || [[ "$peer_addrs3" -lt 2 ]]; then - log_error "Peer discovery via bootnode failed: peer_addrs1=$peer_addrs1, peer_addrs2=$peer_addrs2, peer_addrs3=$peer_addrs3" - test_passed=false - fi - - # Verify KvStore sync on all nodes - if [[ "$kv1" -lt 1 ]] || [[ "$kv2" -lt 1 ]] || [[ "$kv3" -lt 1 ]]; then - log_error "KvStore sync failed: kv1=$kv1, kv2=$kv2, kv3=$kv3" - test_passed=false - fi - - # Verify ProxyState sync on all nodes - if [[ "$ps1" -lt 1 ]] || [[ "$ps2" -lt 1 ]] || [[ "$ps3" -lt 1 ]]; then - log_error "ProxyState sync failed: ps1=$ps1, ps2=$ps2, ps3=$ps3" - test_passed=false - fi - - # Verify consistency on each node - if [[ "$kv1" -ne "$ps1" ]] || [[ "$kv2" -ne "$ps2" ]] || [[ "$kv3" -ne "$ps3" ]]; then - log_error "Inconsistency detected between KvStore and ProxyState" - test_passed=false - fi - - if [[ "$test_passed" == "true" ]]; then - log_info "Three-node bootnode cluster test PASSED" - return 0 - else - log_info "Sync data from node 1: $(debug_get_sync_data $debug_port1)" - log_info "Sync data from node 2: $(debug_get_sync_data $debug_port2)" - log_info "Sync data from node 3: $(debug_get_sync_data $debug_port3)" - return 1 - fi -} - -# ============================================================================= -# Test 14: Node ID reuse rejection -# ============================================================================= -test_node_id_reuse_rejected() { - log_info "========== Node ID Reuse Rejection and Recovery ==========" - cleanup - - # Clean up all state files to ensure fresh start - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node1.json" "$RUN_DIR/gateway-state-node2.json" - - # Start node 1 and node 2, let them sync - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local debug_port1=13015 - local debug_port2=13025 - local admin_port1=13016 - - # Wait for initial sync - log_info "Waiting for initial sync between node 1 and node 2..." - sleep 10 - - # Verify both nodes have synced - if ! has_peer_addr $debug_port1 2; then - log_error "Node 1 missing peer_addr for node 2" - return 1 - fi - if ! has_peer_addr $debug_port2 1; then - log_error "Node 2 missing peer_addr for node 1" - return 1 - fi - log_info "Initial sync completed successfully" - verify_register_response "$(debug_register_cvm $debug_port1 \ - "$(test_public_key 400)" "reuse_app" "reuse_fixture")" >/dev/null || return 1 - wait_for_instances $debug_port2 1 10 || { - log_error "Node 2 did not receive the recovery fixture"; return 1; } - local old_uuid=$(get_node_uuid 13026) - - # Get initial key count on node 1 - local keys_before=$(get_n_keys $admin_port1) - log_info "Keys on node 1 before node 2 restart: $keys_before" - - # Stop node 2 and delete its data (simulating a fresh node trying to reuse the ID) - log_info "Stopping node 2 and deleting its data..." - stop_node 2 - rm -rf "$RUN_DIR/wavekv_node2" - rm -f "$RUN_DIR/gateway-state-node2.json" - - # Restart node 2 - it will have a new UUID but same node_id - log_info "Restarting node 2 with fresh data (new UUID, same node_id)..." - start_node 2 - local new_uuid=$(get_node_uuid 13026) - if [[ -z "$old_uuid" || -z "$new_uuid" || "$old_uuid" == "$new_uuid" ]]; then - log_error "Fresh node 2 did not receive a new UUID" - return 1 - fi - - # Re-register peers - setup_peers 1 2 - - # The first exchange must reject the conflicting identity. A response from the - # established peer then carries the fresh identity record so subsequent rounds - # can converge instead of leaving the pair permanently wedged. - local log_file1="${LOG_DIR}/${CURRENT_TEST}_node1.log" - local log_file2="${LOG_DIR}/${CURRENT_TEST}_node2.log" - local mismatch_seen=false - for _ in $(seq 1 15); do - if grep -q "UUID mismatch" "$log_file1" "$log_file2" 2>/dev/null; then - mismatch_seen=true - break - fi - sleep 1 - done - if [[ "$mismatch_seen" != "true" ]]; then - log_error "Reused node ID was not rejected" - return 1 - fi - - # Node 1 should have rejected sync from new node 2 - # Check if node 1's data is still intact (keys should not decrease) - local keys_after=$(get_n_keys $admin_port1) - log_info "Keys on node 1 after node 2 restart: $keys_after" - - # Verify node 1's data is intact - if [[ "$keys_after" -lt "$keys_before" ]]; then - log_error "Node 1 lost data after node 2 restart with reused ID" - return 1 - fi - - wait_for_instances $debug_port2 1 20 || { - log_error "Fresh node did not recover after the UUID rejection"; return 1; } - wait_for_digest_match persistent 13016 13026 15 || { - log_error "Stores did not converge after UUID recovery"; return 1; } - verify_register_response "$(debug_register_cvm $debug_port2 \ - "$(test_public_key 401)" "reuse_app" "post_recovery")" >/dev/null || return 1 - wait_for_instances $debug_port1 2 15 || { - log_error "Post-recovery write did not propagate"; return 1; } - wait_for_digest_match persistent 13016 13026 10 || return 1 - log_info "Node ID reuse rejection and recovery test PASSED" - return 0 -} - -# ============================================================================= -# Test 15: Periodic persistence -# ============================================================================= -test_periodic_persistence() { - log_info "========== Test 15: Periodic Persistence ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local debug_port=13015 - local admin_port=13016 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register some clients to create data - log_info "Registering clients to create data..." - local success_count=0 - for i in $(seq 1 3); do - local key=$(test_public_key $((900 + i))) - local response=$(debug_register_cvm $debug_port "$key" "persist_app$i" "persist_inst$i") - if verify_register_response "$response" >/dev/null 2>&1; then - ((success_count++)) - fi - done - log_info "Registered $success_count/3 clients" - - if [[ "$success_count" -ne 3 ]]; then - log_error "Failed to register all clients" - return 1 - fi - - # Get initial key count - local keys_before=$(get_n_keys $admin_port) - log_info "Keys before waiting for persist: $keys_before" - - # Wait for periodic persistence (persist_interval is 5s in test config) - log_info "Waiting for periodic persistence (8s)..." - sleep 8 - - # Check log for periodic persist message - local log_file="${LOG_DIR}/${CURRENT_TEST}_node1.log" - if grep -q "periodic persist completed" "$log_file" 2>/dev/null; then - log_info "Found periodic persist message in log" - else - log_error "Periodic persist message not found in log - test FAILED" - return 1 - fi - - # Stop node - stop_node 1 - - # Check WAL file exists and has content - local wal_file="$RUN_DIR/wavekv_node1/node_1.wal" - if [[ ! -f "$wal_file" ]]; then - log_error "WAL file not found: $wal_file" - return 1 - fi - - local wal_size=$(stat -c%s "$wal_file" 2>/dev/null || stat -f%z "$wal_file" 2>/dev/null) - log_info "WAL file size after periodic persist: $wal_size bytes" - - # Restart node and verify data is recovered - log_info "Restarting node to verify persistence..." - start_node 1 - - local keys_after=$(get_n_keys $admin_port) - log_info "Keys after restart: $keys_after" - - if [[ "$keys_after" -ge "$keys_before" ]]; then - log_info "Periodic persistence test PASSED" - return 0 - else - log_error "Periodic persistence test FAILED: keys_before=$keys_before, keys_after=$keys_after" - return 1 - fi -} - -# ============================================================================= -# Admin RPC helper functions -# ============================================================================= - -# Call Admin.SetNodeUrl RPC -# Usage: admin_set_node_url -admin_set_node_url() { - local admin_port=$1 - local node_id=$2 - local url=$3 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.SetNodeUrl" \ - -H "Content-Type: application/json" \ - -d "{\"id\": $node_id, \"url\": \"$url\"}" 2>/dev/null -} - -# Register peers between nodes via Admin RPC -# This is needed since we removed peer_node_ids/peer_urls from config -# Usage: setup_peers -# Example: setup_peers 1 2 3 # Sets up peers between nodes 1, 2, and 3 -setup_peers() { - local node_ids=("$@") - - for src_node in "${node_ids[@]}"; do - local src_admin_port=$((13000 + src_node * 10 + 6)) - - for dst_node in "${node_ids[@]}"; do - if [[ "$src_node" != "$dst_node" ]]; then - local dst_rpc_port=$((13000 + dst_node * 10 + 2)) - local dst_url="https://localhost:${dst_rpc_port}" - admin_set_node_url "$src_admin_port" "$dst_node" "$dst_url" - fi - done - done - - # Wait for peers to be registered - sleep 1 -} - -# Call Admin.SetNodeStatus RPC -# Usage: admin_set_node_status -# status: "up" or "down" -admin_set_node_status() { - local admin_port=$1 - local node_id=$2 - local status=$3 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.SetNodeStatus" \ - -H "Content-Type: application/json" \ - -d "{\"id\": $node_id, \"status\": \"$status\"}" 2>/dev/null -} - -# Call Admin.Status RPC to get all nodes -# Usage: admin_get_status -admin_get_status() { - local admin_port=$1 - curl -s -X POST "http://localhost:${admin_port}/prpc/Admin.Status" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -# Get peer URL from sync data -# Usage: get_peer_url -get_peer_url_from_sync() { - local debug_port=$1 - local node_id=$2 - local response=$(debug_get_sync_data "$debug_port") - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - for pa in d.get('peer_addrs', []): - if pa.get('node_id') == $node_id: - print(pa.get('url', '')) - sys.exit(0) - print('') -except Exception: - print('') -" 2>/dev/null -} - -# ============================================================================= -# Test 16: Admin.SetNodeUrl RPC -# ============================================================================= -test_admin_set_node_url() { - log_info "========== Test 16: Admin.SetNodeUrl RPC ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Set URL for a new node (node 2) via Admin RPC - local new_url="https://new-node2.example.com:8011" - log_info "Setting node 2 URL via Admin.SetNodeUrl..." - local response=$(admin_set_node_url $admin_port 2 "$new_url") - log_info "SetNodeUrl response: $response" - - # Check if the response contains an error - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeUrl returned error: $response" - return 1 - fi - - # Wait for data to be written - sleep 2 - - # Verify the URL was stored in KvStore - local stored_url=$(get_peer_url_from_sync $debug_port 2) - log_info "Stored URL for node 2: $stored_url" - - if [[ "$stored_url" == "$new_url" ]]; then - log_info "Admin.SetNodeUrl test PASSED" - return 0 - else - log_error "Admin.SetNodeUrl test FAILED: expected '$new_url', got '$stored_url'" - log_info "Sync data: $(debug_get_sync_data $debug_port)" - return 1 - fi -} - -# ============================================================================= -# Test 17: Admin.SetNodeStatus RPC -# ============================================================================= -test_admin_set_node_status() { - log_info "========== Test 17: Admin.SetNodeStatus RPC ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # First set a URL for node 2 so we have a peer - admin_set_node_url $admin_port 2 "https://node2.example.com:8011" - sleep 1 - - # Set node 2 status to "down" - log_info "Setting node 2 status to 'down' via Admin.SetNodeStatus..." - local response=$(admin_set_node_status $admin_port 2 "down") - log_info "SetNodeStatus response: $response" - - # Check if the response contains an error - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeStatus returned error: $response" - return 1 - fi - - sleep 1 - - # Set node 2 status back to "up" - log_info "Setting node 2 status to 'up' via Admin.SetNodeStatus..." - response=$(admin_set_node_status $admin_port 2 "up") - log_info "SetNodeStatus response: $response" - - if echo "$response" | grep -q '"error"'; then - log_error "SetNodeStatus returned error: $response" - return 1 - fi - - # Test invalid status - log_info "Testing invalid status..." - response=$(admin_set_node_status $admin_port 2 "invalid") - if echo "$response" | grep -q '"error"'; then - log_info "Invalid status correctly rejected" - else - log_warn "Invalid status was not rejected (may be acceptable)" - fi - - log_info "Admin.SetNodeStatus test PASSED" - return 0 -} - -# ============================================================================= -# Test 18: Node down excluded from RegisterCvm response -# ============================================================================= -test_node_status_register_exclude() { - log_info "========== Test 18: Node Down Excluded from Registration ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" "$RUN_DIR/wavekv_node2" - - generate_config 1 - generate_config 2 - - start_node 1 - start_node 2 - - # Register peers so nodes can discover each other - setup_peers 1 2 - - local admin_port1=13016 - local admin_port2=13026 - local debug_port1=13015 - - # Wait for sync - sleep 5 - - # Verify debug service is available - if ! check_debug_service $debug_port1; then - log_error "Debug service not available on node 1" - return 1 - fi - - # Set node 2 status to "down" via node 1's admin API - log_info "Setting node 2 status to 'down'..." - admin_set_node_status $admin_port1 2 "down" - sleep 2 - - # Register a client on node 1 - log_info "Registering client on node 1 (node 2 is down)..." - local response=$(debug_register_cvm $debug_port1 "$(test_public_key 505)" "downtest_app" "downtest_inst") - log_info "Register response: $response" - - # Verify registration succeeded - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed" - return 1 - fi - log_info "Registered client with IP: $client_ip" - - # Check gateways list in response - should NOT include node 2 - local has_node2=$(echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - gateways = d.get('gateways', []) - for gw in gateways: - if gw.get('id') == 2: - sys.exit(0) - sys.exit(1) -except Exception: - sys.exit(1) -" && echo "yes" || echo "no") - - if [[ "$has_node2" == "yes" ]]; then - log_error "Node 2 (down) was included in registration response" - log_info "Response: $response" - return 1 - else - log_info "Node 2 (down) correctly excluded from registration response" - fi - - # Set node 2 status back to "up" - log_info "Setting node 2 status to 'up'..." - admin_set_node_status $admin_port1 2 "up" - sleep 2 - - # Register another client - log_info "Registering client on node 1 (node 2 is now up)..." - response=$(debug_register_cvm $debug_port1 "$(test_public_key 506)" "uptest_app" "uptest_inst2") - - # Check gateways list - should now include node 2 - has_node2=$(echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - gateways = d.get('gateways', []) - for gw in gateways: - if gw.get('id') == 2: - sys.exit(0) - sys.exit(1) -except Exception: - sys.exit(1) -" && echo "yes" || echo "no") - - if [[ "$has_node2" == "no" ]]; then - log_error "Node 2 (up) was NOT included in registration response" - log_info "Response: $response" - return 1 - else - log_info "Node 2 (up) correctly included in registration response" - fi - - log_info "Node down excluded from registration test PASSED" - return 0 -} - -# ============================================================================= -# Test 19: Node down rejects RegisterCvm requests -# ============================================================================= -test_node_status_register_reject() { - log_info "========== Test 19: Node Down Rejects Registration ==========" - cleanup - - rm -rf "$RUN_DIR/wavekv_node1" - - generate_config 1 - start_node 1 - - local admin_port=13016 - local debug_port=13015 - - # Verify debug service is available - if ! check_debug_service $debug_port; then - log_error "Debug service not available" - return 1 - fi - - # Register a client when node is up (should succeed) - log_info "Registering client when node 1 is up..." - local response=$(debug_register_cvm $debug_port "$(test_public_key 507)" "upnode_app" "upnode_inst") - local client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed when node was up" - return 1 - fi - log_info "Registration succeeded when node was up (IP: $client_ip)" - - # Set node 1 status to "down" (marking itself as down) - log_info "Setting node 1 status to 'down'..." - admin_set_node_status $admin_port 1 "down" - sleep 2 - - # Try to register a client when node is down (should fail) - log_info "Attempting to register client when node 1 is down..." - response=$(debug_register_cvm $debug_port "$(test_public_key 508)" "downnode_app" "downnode_inst") - log_info "Register response: $response" - - # Check if response contains error about node being down - if echo "$response" | grep -qi "error"; then - log_info "Registration correctly rejected when node is down" - if echo "$response" | grep -qi "marked as down"; then - log_info "Error message mentions 'marked as down' (correct)" - fi - else - log_error "Registration was NOT rejected when node is down" - log_info "Response: $response" - return 1 - fi - - # Set node 1 status back to "up" - log_info "Setting node 1 status to 'up'..." - admin_set_node_status $admin_port 1 "up" - sleep 2 - - # Register a client again (should succeed) - log_info "Registering client when node 1 is back up..." - response=$(debug_register_cvm $debug_port "$(test_public_key 509)" "backup_app" "backup_inst") - client_ip=$(verify_register_response "$response") - if [[ -z "$client_ip" ]]; then - log_error "Registration failed when node was back up" - return 1 - fi - log_info "Registration succeeded when node was back up (IP: $client_ip)" - - log_info "Node down rejects registration test PASSED" - return 0 -} - -# ============================================================================= -# Clean command - remove all generated files -# ============================================================================= -clean() { - log_info "Cleaning up generated files..." - - # Kill only test gateway processes (matching our specific config path) - pkill -9 -f "dstack-gateway -c ${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - pkill -9 -f "dstack-gateway.*${SCRIPT_DIR}/${RUN_DIR}/node" >/dev/null 2>&1 || true - sleep 1 - - # Remove WireGuard interfaces (only our test interfaces need sudo) - sudo ip link delete wavekv-test1 2>/dev/null || true - sudo ip link delete wavekv-test2 2>/dev/null || true - sudo ip link delete wavekv-test3 2>/dev/null || true - - # Remove run directory (contains all generated files including certs) - rm -rf "$RUN_DIR" - - log_info "Cleanup complete" -} - -# ============================================================================= -# Ensure all certificates exist (CA + RPC + proxy) -# ============================================================================= -ensure_certs() { - # Create directories - mkdir -p "$CERTS_DIR" - mkdir -p "$RUN_DIR/certbot/live" - - # Generate CA certificate if not exists - if [[ ! -f "$CERTS_DIR/gateway-ca.key" ]] || [[ ! -f "$CERTS_DIR/gateway-ca.cert" ]]; then - log_info "Creating CA certificate..." - openssl genrsa -out "$CERTS_DIR/gateway-ca.key" 2048 2>/dev/null - openssl req -x509 -new -nodes \ - -key "$CERTS_DIR/gateway-ca.key" \ - -sha256 -days 365 \ - -out "$CERTS_DIR/gateway-ca.cert" \ - -subj "/CN=Test CA/O=WaveKV Test" \ - 2>/dev/null - fi - - # Generate RPC certificate signed by CA if not exists - if [[ ! -f "$CERTS_DIR/gateway-rpc.key" ]] || [[ ! -f "$CERTS_DIR/gateway-rpc.cert" ]]; then - log_info "Creating RPC certificate signed by CA..." - openssl genrsa -out "$CERTS_DIR/gateway-rpc.key" 2048 2>/dev/null - openssl req -new \ - -key "$CERTS_DIR/gateway-rpc.key" \ - -out "$CERTS_DIR/gateway-rpc.csr" \ - -subj "/CN=localhost" \ - 2>/dev/null - # Create certificate with SAN for localhost - cat >"$CERTS_DIR/ext.cnf" </dev/null - rm -f "$CERTS_DIR/gateway-rpc.csr" "$CERTS_DIR/ext.cnf" - fi - - # Generate proxy certificates (for TLS termination) - local proxy_cert_dir="$RUN_DIR/certbot/live" - if [[ ! -f "$proxy_cert_dir/cert.pem" ]] || [[ ! -f "$proxy_cert_dir/key.pem" ]]; then - log_info "Creating proxy certificates..." - openssl req -x509 -newkey rsa:2048 -nodes \ - -keyout "$proxy_cert_dir/key.pem" \ - -out "$proxy_cert_dir/cert.pem" \ - -days 365 \ - -subj "/CN=localhost" \ - 2>/dev/null - fi -} - -# ============================================================================= -# Main -# ============================================================================= -main() { - # Handle clean command - if [[ "${1:-}" == "clean" ]]; then - clean - exit 0 - fi - - # Handle cfg command - generate node configuration - if [[ "${1:-}" == "cfg" ]]; then - local node_id="${2:-}" - if [[ -z "$node_id" ]]; then - log_error "Usage: $0 cfg " - log_info "Example: $0 cfg 1" - exit 1 - fi - - # Ensure certificates exist - ensure_certs - - # Generate config for the specified node - generate_config "$node_id" - log_info "Configuration generated: $RUN_DIR/node${node_id}.toml" - exit 0 - fi - - # Handle ls command - list all test cases - if [[ "${1:-}" == "ls" ]]; then - echo "Available test cases:" - echo "" - echo "Quick tests:" - echo " test_persistence - Single node persistence" - echo " test_status_endpoint - Status endpoint structure" - echo " test_prpc_register - prpc DebugRegisterCvm endpoint" - echo " test_prpc_info - prpc Info endpoint" - echo " test_wal_integrity - WAL file integrity" - echo "" - echo "Sync tests:" - echo " test_multi_node_sync - Multi-node sync" - echo " test_node_recovery - Node recovery after disconnect" - echo " test_cross_node_data_sync - Cross-node data sync verification" - echo " test_push_fast_path - Push propagation before periodic sync" - echo " test_periodic_repair_after_missed_push - Periodic repair after a missed push" - echo " test_bootstrap_after_data_dir_loss - Bootstrap after local store loss" - echo " test_divergent_partition_writes - Merge writes from divergent partitions" - echo " test_push_periodic_overlap - Push racing periodic synchronization" - echo " test_delayed_bootnode_recovery - Bootnode discovery retry" - echo " test_interrupted_sync_recovery - Recovery after interrupted sync" - echo " test_ephemeral_recovery - Ephemeral convergence after restart" - echo " test_partial_cluster_bootstrap - Bootstrap with one cluster peer down" - echo " test_node_id_reuse_rejected - Node ID conflict rejection and recovery" - echo "" - echo "Advanced tests:" - echo " test_client_registration_persistence - Client registration and persistence" - echo " test_stress_writes - Stress test - multiple writes" - echo " test_network_partition - Network partition simulation" - echo " test_three_node_cluster - Three-node cluster" - echo " test_three_node_bootnode - Three-node cluster with bootnode" - echo " test_periodic_persistence - Periodic persistence" - echo "" - echo "Admin RPC tests:" - echo " test_admin_set_node_url - Admin.SetNodeUrl RPC" - echo " test_admin_set_node_status - Admin.SetNodeStatus RPC" - echo " test_node_status_register_exclude - Node down excluded from registration" - echo " test_node_status_register_reject - Node down rejects registration" - echo "" - echo "Usage:" - echo " $0 - Run all tests" - echo " $0 quick - Run quick tests only" - echo " $0 sync - Run sync tests only" - echo " $0 advanced - Run advanced tests only" - echo " $0 admin - Run admin RPC tests only" - echo " $0 case - Run specific test case" - echo " $0 ls - List all test cases" - echo " $0 clean - Clean up generated files" - exit 0 - fi - - # Handle case command - run specific test case - if [[ "${1:-}" == "case" ]]; then - local test_case="${2:-}" - if [[ -z "$test_case" ]]; then - log_error "Usage: $0 case " - log_info "Run '$0 ls' to see all available test cases" - exit 1 - fi - - # Check if gateway binary exists - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_error "Gateway binary not found: $GATEWAY_BIN" - log_info "Please run: cargo build --release" - exit 1 - fi - - # Ensure certificates exist - ensure_certs - - # Check if test function exists - if ! declare -f "$test_case" >/dev/null; then - log_error "Test case not found: $test_case" - log_info "Use '$0 case' to see available test cases" - exit 1 - fi - - # Run the specific test - log_info "Running test case: $test_case" - CURRENT_TEST="$test_case" - if $test_case; then - log_info "Test PASSED: $test_case" - cleanup - exit 0 - else - log_error "Test FAILED: $test_case" - cleanup - exit 1 - fi - fi - - log_info "Starting WaveKV integration tests..." - - if [[ ! -f "$GATEWAY_BIN" ]]; then - log_error "Gateway binary not found: $GATEWAY_BIN" - log_info "Please run: cargo build --release" - exit 1 - fi - - # Ensure all certificates exist (RPC + proxy) - ensure_certs - - local failed=0 - local passed=0 - local failed_tests=() - - run_test() { - local test_name=$1 - CURRENT_TEST="$test_name" - if $test_name; then - ((passed++)) - else - ((failed++)) - failed_tests+=("$test_name") - fi - cleanup - } - - # Run selected test or all tests - local test_filter="${1:-all}" - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "quick" ]]; then - run_test test_persistence - run_test test_status_endpoint - run_test test_prpc_register - run_test test_prpc_info - run_test test_wal_integrity - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "sync" ]]; then - run_test test_multi_node_sync - run_test test_node_recovery - run_test test_cross_node_data_sync - run_test test_push_fast_path - run_test test_periodic_repair_after_missed_push - run_test test_bootstrap_after_data_dir_loss - run_test test_divergent_partition_writes - run_test test_push_periodic_overlap - run_test test_delayed_bootnode_recovery - run_test test_interrupted_sync_recovery - run_test test_ephemeral_recovery - run_test test_partial_cluster_bootstrap - run_test test_node_id_reuse_rejected - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "advanced" ]]; then - run_test test_client_registration_persistence - run_test test_stress_writes - run_test test_network_partition - run_test test_three_node_cluster - run_test test_three_node_bootnode - run_test test_periodic_persistence - fi - - if [[ "$test_filter" == "all" ]] || [[ "$test_filter" == "admin" ]]; then - run_test test_admin_set_node_url - run_test test_admin_set_node_status - run_test test_node_status_register_exclude - run_test test_node_status_register_reject - fi - - echo "" - log_info "==========================================" - log_info "Tests passed: $passed" - if [[ $failed -gt 0 ]]; then - log_error "Tests failed: $failed" - echo "" - log_error "Failed test cases:" - for test_name in "${failed_tests[@]}"; do - log_error " - $test_name" - done - echo "" - log_info "To rerun a failed test:" - log_info " $0 case " - log_info "Example:" - if [[ ${#failed_tests[@]} -gt 0 ]]; then - log_info " $0 case ${failed_tests[0]}" - fi - fi - log_info "==========================================" - - return $failed -} - -# Run if executed directly -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - main "$@" -fi From 007d478f05d5ff73a58461ca1c7078e7d6eb33fc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:47:52 -0700 Subject: [PATCH 05/27] test(gateway): delete the certbot suite the e2e run covers `test_certbot.sh` could not be containerised without keeping what made it unrunnable: it needs a real Let's Encrypt staging account and real Cloudflare credentials, so it was never in CI and could only be run by hand, by someone holding both. Its four assertions -- an account is created, an order completes, the certificate lands, a renewal replaces it -- are made more strongly by e2e phases 5 and 6, against Pebble and the mock DNS API, across three nodes rather than one, and on every push. --- dstack/gateway/test-run/test_certbot.sh | 562 ------------------------ 1 file changed, 562 deletions(-) delete mode 100755 dstack/gateway/test-run/test_certbot.sh diff --git a/dstack/gateway/test-run/test_certbot.sh b/dstack/gateway/test-run/test_certbot.sh deleted file mode 100755 index 29d3a58b4..000000000 --- a/dstack/gateway/test-run/test_certbot.sh +++ /dev/null @@ -1,562 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Distributed Certbot E2E test script -# Tests certificate issuance and synchronization across gateway nodes - -set -m - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# Show help -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "" - echo "Distributed Certbot E2E Test" - echo "" - echo "Options:" - echo " --fresh Clean everything and request new certificate from ACME" - echo " --sync-only Keep existing cert, only test sync between nodes" - echo " --clean Clean all test data and exit" - echo " -h, --help Show this help message" - echo "" - echo "Default (no options): Keep ACME account, request new certificate" - echo "" - echo "Examples:" - echo " $0 # Keep account, new cert" - echo " $0 --fresh # Fresh start, new account and cert" - echo " $0 --sync-only # Test sync with existing cert" - echo " $0 --clean # Clean up all test data" -} - -# Parse arguments -MODE="default" -while [[ $# -gt 0 ]]; do - case $1 in - --fresh) - MODE="fresh" - shift - ;; - --sync-only) - MODE="sync-only" - shift - ;; - --clean) - MODE="clean" - shift - ;; - -h|--help) - show_help - exit 0 - ;; - *) - echo "Unknown option: $1" - show_help - exit 1 - ;; - esac -done - -# Load environment variables from .env -if [[ -f ".env" ]]; then - source ".env" -else - echo "ERROR: .env file not found!" - echo "" - echo "Please create a .env file with the following variables:" - echo " CF_API_TOKEN=" - echo " CF_ZONE_ID=" - echo " TEST_DOMAIN=" - echo "" - echo "The domain must be managed by Cloudflare and the API token must have" - echo "permissions to manage DNS records and CAA records." - exit 1 -fi - -# Validate required environment variables -if [[ -z "$CF_API_TOKEN" ]]; then - echo "ERROR: CF_API_TOKEN is not set in .env" - exit 1 -fi - -if [[ -z "$CF_ZONE_ID" ]]; then - echo "ERROR: CF_ZONE_ID is not set in .env" - exit 1 -fi - -if [[ -z "$TEST_DOMAIN" ]]; then - echo "ERROR: TEST_DOMAIN is not set in .env" - exit 1 -fi - -GATEWAY_BIN="$SCRIPT_DIR/../../target/release/dstack-gateway" -RUN_DIR="run" -CERTS_DIR="$RUN_DIR/certs" -CA_CERT="$CERTS_DIR/gateway-ca.cert" -LOG_DIR="$RUN_DIR/logs" -CURRENT_TEST="test_certbot" - -# Let's Encrypt staging URL (for testing without rate limits) -ACME_STAGING_URL="https://acme-staging-v02.api.letsencrypt.org/directory" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -cleanup() { - log_info "Cleaning up..." - sudo pkill -9 -f "dstack-gateway.*certbot_node[12].toml" >/dev/null 2>&1 || true - sudo ip link delete certbot-test1 2>/dev/null || true - sudo ip link delete certbot-test2 2>/dev/null || true - sleep 1 - stty sane 2>/dev/null || true -} - -trap cleanup EXIT - -# Generate node config with certbot enabled -generate_certbot_config() { - local node_id=$1 - local rpc_port=$((14000 + node_id * 10 + 2)) - local wg_port=$((14000 + node_id * 10 + 3)) - local proxy_port=$((14000 + node_id * 10 + 4)) - local debug_port=$((14000 + node_id * 10 + 5)) - local wg_ip="10.0.4${node_id}.1/24" - - # Build peer config - local other_node=$((3 - node_id)) # If node_id=1, other=2; if node_id=2, other=1 - local other_rpc_port=$((14000 + other_node * 10 + 2)) - - local abs_run_dir="$SCRIPT_DIR/$RUN_DIR" - local certbot_dir="$abs_run_dir/certbot_node${node_id}" - - mkdir -p "$certbot_dir" - - cat > "$RUN_DIR/certbot_node${node_id}.toml" << EOF -log_level = "info" -address = "0.0.0.0" -port = ${rpc_port} - -[tls] -key = "${abs_run_dir}/certs/gateway-rpc.key" -certs = "${abs_run_dir}/certs/gateway-rpc.cert" - -[tls.mutual] -ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" -mandatory = false - -[core] -rpc_domain = "" - -[core.debug] -insecure_enable_debug_rpc = true -insecure_skip_attestation = true -port = ${debug_port} -address = "127.0.0.1" - -[core.sync] -enabled = true -interval = "5s" -timeout = "10s" -my_url = "https://localhost:${rpc_port}" -bootnode = "https://localhost:${other_rpc_port}" -node_id = ${node_id} -data_dir = "${RUN_DIR}/wavekv_certbot_node${node_id}" - -[core.certbot] -enabled = true -workdir = "${certbot_dir}" -acme_url = "${ACME_STAGING_URL}" -cf_api_token = "${CF_API_TOKEN}" -cf_zone_id = "${CF_ZONE_ID}" -auto_set_caa = true -domain = "${TEST_DOMAIN}" -renew_interval = "1h" -renew_before_expiration = "720h" -renew_timeout = "5m" - -[core.wg] -private_key = "SEcoI37oGWynhukxXo5Mi8/8zZBU6abg6T1TOJRMj1Y=" -public_key = "xc+7qkdeNFfl4g4xirGGGXHMc0cABuE5IHaLeCASVWM=" -listen_port = ${wg_port} -ip = "${wg_ip}" -reserved_net = ["10.0.4${node_id}.1/31"] -client_ip_range = "10.0.4${node_id}.1/24" -config_path = "${RUN_DIR}/wg_certbot_node${node_id}.conf" -interface = "certbot-test${node_id}" -endpoint = "127.0.0.1:${wg_port}" - -[core.proxy] -cert_chain = "${certbot_dir}/live/cert.pem" -cert_key = "${certbot_dir}/live/key.pem" -base_domain = "tdxlab.dstack.org" -listen_addr = "0.0.0.0" -listen_port = ${proxy_port} -tappd_port = 8090 -external_port = ${proxy_port} -EOF - log_info "Generated certbot_node${node_id}.toml (rpc=${rpc_port}, debug=${debug_port}, proxy=${proxy_port})" -} - -start_certbot_node() { - local node_id=$1 - local config="$RUN_DIR/certbot_node${node_id}.toml" - local log_file="${LOG_DIR}/${CURRENT_TEST}_node${node_id}.log" - - log_info "Starting certbot node ${node_id}..." - mkdir -p "$RUN_DIR/wavekv_certbot_node${node_id}" - mkdir -p "$LOG_DIR" - ( sudo RUST_LOG=info "$GATEWAY_BIN" -c "$config" > "$log_file" 2>&1 & ) - - # Wait for process to either stabilize or fail - local max_wait=30 - local waited=0 - while [[ $waited -lt $max_wait ]]; do - sleep 2 - waited=$((waited + 2)) - - if ! pgrep -f "dstack-gateway.*${config}" > /dev/null; then - # Process exited, check why - log_error "Certbot node ${node_id} exited after ${waited}s" - echo "--- Log output ---" - cat "$log_file" - echo "--- End log ---" - - # Check for rate limit error - if grep -q "rateLimited" "$log_file"; then - log_error "Let's Encrypt rate limit hit. Wait a few minutes and retry." - fi - return 1 - fi - - # Check if cert files exist (indicates successful init) - local certbot_dir="$RUN_DIR/certbot_node${node_id}" - if [[ -f "$certbot_dir/live/cert.pem" ]] && [[ -f "$certbot_dir/live/key.pem" ]]; then - log_info "Certbot node ${node_id} started and certificate obtained" - return 0 - fi - - log_info "Waiting for node ${node_id} to initialize... (${waited}s)" - done - - # Process still running but no cert yet - might still be requesting - if pgrep -f "dstack-gateway.*${config}" > /dev/null; then - log_info "Certbot node ${node_id} still running, certificate request in progress" - return 0 - fi - - log_error "Certbot node ${node_id} failed to start within ${max_wait}s" - cat "$log_file" - return 1 -} - -stop_certbot_node() { - local node_id=$1 - log_info "Stopping certbot node ${node_id}..." - sudo pkill -9 -f "dstack-gateway.*certbot_node${node_id}.toml" >/dev/null 2>&1 || true - sleep 1 -} - -# Get debug sync data from a node -debug_get_sync_data() { - local debug_port=$1 - curl -s "http://localhost:${debug_port}/prpc/GetSyncData" \ - -H "Content-Type: application/json" \ - -d '{}' 2>/dev/null -} - -# Check if KvStore has cert data for the domain -check_kvstore_cert() { - local debug_port=$1 - local response=$(debug_get_sync_data "$debug_port") - - # The cert data would be in the persistent store - # For now, check if we can get any data - if [[ -z "$response" ]]; then - return 1 - fi - - # Check for cert-related keys in the response - echo "$response" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) - # Check if there are any keys that start with 'cert/' - # This is a simplified check - print('ok') - sys.exit(0) -except Exception as e: - print(f'error: {e}', file=sys.stderr) - sys.exit(1) -" 2>/dev/null -} - -# Check if proxy is using a valid certificate by connecting via TLS -check_proxy_cert() { - local proxy_port=$1 - - # Use gateway.{base_domain} as the SNI for health endpoint - local gateway_host="gateway.tdxlab.dstack.org" - - # Use openssl to check the certificate - local cert_info=$(echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null) - - if [[ -z "$cert_info" ]]; then - log_error "Failed to connect to proxy on port ${proxy_port}" - return 1 - fi - - # Check if the certificate is valid (not self-signed test cert) - # For staging certs, the issuer should contain "Staging" or "(STAGING)" - local issuer=$(echo "$cert_info" | openssl x509 -noout -issuer 2>/dev/null) - - if echo "$issuer" | grep -qi "staging\|fake\|test"; then - log_info "Proxy on port ${proxy_port} is using Let's Encrypt staging certificate" - log_info "Issuer: $issuer" - return 0 - elif echo "$issuer" | grep -qi "let's encrypt\|letsencrypt"; then - log_info "Proxy on port ${proxy_port} is using Let's Encrypt certificate" - log_info "Issuer: $issuer" - return 0 - else - log_warn "Proxy on port ${proxy_port} certificate issuer: $issuer" - # Still return success if we got a certificate - return 0 - fi -} - -# Get certificate expiry from proxy health endpoint -get_proxy_cert_expiry() { - local proxy_port=$1 - # Use gateway.{base_domain} as the SNI for health endpoint - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -enddate 2>/dev/null | \ - cut -d= -f2 -} - -# Get certificate serial from proxy health endpoint -get_proxy_cert_serial() { - local proxy_port=$1 - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -serial 2>/dev/null | \ - cut -d= -f2 -} - -# Get certificate issuer from proxy -get_proxy_cert_issuer() { - local proxy_port=$1 - local gateway_host="gateway.tdxlab.dstack.org" - echo | timeout 5 openssl s_client -connect "localhost:${proxy_port}" -servername "$gateway_host" 2>/dev/null | \ - openssl x509 -noout -issuer 2>/dev/null -} - -# Wait for certificate to be issued (with timeout) -wait_for_cert() { - local proxy_port=$1 - local timeout_secs=${2:-300} # Default 5 minutes - local start_time=$(date +%s) - - log_info "Waiting for certificate to be issued (timeout: ${timeout_secs}s)..." - - while true; do - local current_time=$(date +%s) - local elapsed=$((current_time - start_time)) - - if [[ $elapsed -ge $timeout_secs ]]; then - log_error "Timeout waiting for certificate" - return 1 - fi - - # Try to get certificate info - local expiry=$(get_proxy_cert_expiry "$proxy_port") - if [[ -n "$expiry" ]]; then - log_info "Certificate detected! Expiry: $expiry" - return 0 - fi - - log_info "Waiting... (${elapsed}s elapsed)" - sleep 10 - done -} - -# ============================================================ -# Main Test -# ============================================================ - -do_clean() { - log_info "Cleaning all certbot test data..." - cleanup - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - sudo rm -rf "$RUN_DIR/wavekv_certbot_node1" "$RUN_DIR/wavekv_certbot_node2" - sudo rm -f "$RUN_DIR/gateway-state-certbot-node1.json" "$RUN_DIR/gateway-state-certbot-node2.json" - log_info "Done." -} - -main() { - log_info "==========================================" - log_info "Distributed Certbot E2E Test" - log_info "==========================================" - log_info "Test domain: $TEST_DOMAIN" - log_info "ACME URL: $ACME_STAGING_URL" - log_info "Mode: $MODE" - log_info "" - - # Handle --clean mode - if [[ "$MODE" == "clean" ]]; then - do_clean - return 0 - fi - - # Handle --sync-only mode: check if cert exists - if [[ "$MODE" == "sync-only" ]]; then - if [[ ! -f "$RUN_DIR/certbot_node1/live/cert.pem" ]]; then - log_error "No existing certificate found. Run without --sync-only first." - return 1 - fi - log_info "Using existing certificate for sync test" - fi - - # Clean up processes and state - cleanup - - # Decide what to clean based on mode - case "$MODE" in - fresh) - # Clean everything including ACME account - log_info "Fresh mode: cleaning all data including ACME account" - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - ;; - sync-only) - # Keep node1 cert, only clean node2 and wavekv - log_info "Sync-only mode: keeping node1 certificate" - sudo rm -rf "$RUN_DIR/certbot_node2" - ;; - *) - # Default: keep ACME account (credentials.json), clean certs - log_info "Default mode: keeping ACME account, requesting new certificate" - # Backup credentials if exists - if [[ -f "$RUN_DIR/certbot_node1/credentials.json" ]]; then - sudo cp "$RUN_DIR/certbot_node1/credentials.json" /tmp/certbot_credentials_backup.json - fi - sudo rm -rf "$RUN_DIR/certbot_node1" "$RUN_DIR/certbot_node2" - # Restore credentials - if [[ -f /tmp/certbot_credentials_backup.json ]]; then - mkdir -p "$RUN_DIR/certbot_node1" - sudo mv /tmp/certbot_credentials_backup.json "$RUN_DIR/certbot_node1/credentials.json" - fi - ;; - esac - - # Always clean wavekv and gateway state - sudo rm -rf "$RUN_DIR/wavekv_certbot_node1" "$RUN_DIR/wavekv_certbot_node2" - sudo rm -f "$RUN_DIR/gateway-state-certbot-node1.json" "$RUN_DIR/gateway-state-certbot-node2.json" - - # Generate configs - log_info "Generating node configurations..." - generate_certbot_config 1 - generate_certbot_config 2 - - # Start Node 1 first - it will request the certificate - log_info "" - log_info "==========================================" - log_info "Phase 1: Start Node 1 and request certificate" - log_info "==========================================" - - if ! start_certbot_node 1; then - log_error "Failed to start node 1" - return 1 - fi - - # Wait for certificate to be issued - local proxy_port_1=14014 - if ! wait_for_cert "$proxy_port_1" 300; then - log_error "Node 1 failed to obtain certificate" - cat "$LOG_DIR/${CURRENT_TEST}_node1.log" | tail -50 - return 1 - fi - - # Get Node 1's certificate info - local node1_serial=$(get_proxy_cert_serial "$proxy_port_1") - local node1_expiry=$(get_proxy_cert_expiry "$proxy_port_1") - log_info "Node 1 certificate serial: $node1_serial" - log_info "Node 1 certificate expiry: $node1_expiry" - - # Show certificate source logs for Node 1 - log_info "" - log_info "Node 1 certificate source:" - grep -E "cert\[|acme\[" "$LOG_DIR/${CURRENT_TEST}_node1.log" 2>/dev/null | sed 's/^/ /' - - # Start Node 2 - it should sync the certificate from Node 1 - log_info "" - log_info "==========================================" - log_info "Phase 2: Start Node 2 and verify sync" - log_info "==========================================" - - if ! start_certbot_node 2; then - log_error "Failed to start node 2" - return 1 - fi - - # Wait for Node 2 to sync and load the certificate - local proxy_port_2=14024 - sleep 10 # Give time for sync - - if ! wait_for_cert "$proxy_port_2" 60; then - log_error "Node 2 failed to obtain certificate via sync" - cat "$LOG_DIR/${CURRENT_TEST}_node2.log" | tail -50 - return 1 - fi - - # Get Node 2's certificate info - local node2_serial=$(get_proxy_cert_serial "$proxy_port_2") - local node2_expiry=$(get_proxy_cert_expiry "$proxy_port_2") - log_info "Node 2 certificate serial: $node2_serial" - log_info "Node 2 certificate expiry: $node2_expiry" - - # Show certificate source logs for Node 2 - log_info "" - log_info "Node 2 certificate source:" - grep -E "cert\[|acme\[" "$LOG_DIR/${CURRENT_TEST}_node2.log" 2>/dev/null | sed 's/^/ /' - - # Verify both nodes have the same certificate - log_info "" - log_info "==========================================" - log_info "Verification" - log_info "==========================================" - - if [[ "$node1_serial" == "$node2_serial" ]]; then - log_info "SUCCESS: Both nodes have the same certificate (serial: $node1_serial)" - else - log_error "FAILURE: Certificate mismatch!" - log_error " Node 1 serial: $node1_serial" - log_error " Node 2 serial: $node2_serial" - return 1 - fi - - # Check that proxy is actually using the certificate - check_proxy_cert "$proxy_port_1" - check_proxy_cert "$proxy_port_2" - - log_info "" - log_info "==========================================" - log_info "All tests passed!" - log_info "==========================================" - - return 0 -} - -# Run main -main -exit $? From 6b8dda47c416880d4e71b2f694e63a8e53079db1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:48:05 -0700 Subject: [PATCH 06/27] test(gateway): run the proxy suite in a container The suite ran on the host and asked for sudo twice: to create the link the gateway wants at startup, and to `rmmod tls`. It now runs in one container that holds the origin, the probe and the gateway in a single network namespace, which `insecure_localhost_backend` requires -- it resolves an app address to 127.0.0.1, and that has to be the same 127.0.0.1 the origin is on. The suite's ~25 gateway restarts stay inside that container rather than becoming compose lifecycle, so a restart costs what it did before. NET_ADMIN in the container's own namespace replaces the first sudo, and nothing it creates outlives the run. The second is replaced by a seccomp profile that makes `setsockopt(IPPROTO_TCP, TCP_ULP)` return ENOPROTOOPT, which is exactly what `probe_ktls` sees on a kernel built without CONFIG_TLS. `rmmod tls` took the module from the whole host, needed passwordless sudo, and skipped silently whenever anything else on the machine held it -- so the arm that guards against a gated offload returning HTTP 200 with a truncated body mostly did not run. A profile is fixed at container creation, so that arm gets its own container and the main run skips it. --- dstack/gateway/test-run/proxy-e2e/Dockerfile | 24 +++++ dstack/gateway/test-run/proxy-e2e/README.md | 26 +++++ .../test-run/proxy-e2e/docker-compose.yml | 63 ++++++++++++ .../test-run/proxy-e2e/notls-seccomp.json | 14 +++ .../proxy-e2e/notls-seccomp.json.license | 3 + .../test-run/proxy-e2e/run-proxy-tests.sh | 96 +++++++++++++++++++ dstack/gateway/test-run/proxy/gwconfig.py | 1 - dstack/gateway/test-run/test_proxy.sh | 59 +++++++----- 8 files changed, 259 insertions(+), 27 deletions(-) create mode 100644 dstack/gateway/test-run/proxy-e2e/Dockerfile create mode 100644 dstack/gateway/test-run/proxy-e2e/README.md create mode 100644 dstack/gateway/test-run/proxy-e2e/docker-compose.yml create mode 100644 dstack/gateway/test-run/proxy-e2e/notls-seccomp.json create mode 100644 dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license create mode 100755 dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh diff --git a/dstack/gateway/test-run/proxy-e2e/Dockerfile b/dstack/gateway/test-run/proxy-e2e/Dockerfile new file mode 100644 index 000000000..f6852017b --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/Dockerfile @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# The proxy data-path suite runs the gateway, the origin server and the probe +# client together in one container, because they must share a network +# namespace: `insecure_localhost_backend` resolves the backend to 127.0.0.1 as +# seen by the gateway itself. +FROM debian:bookworm-slim + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + --no-install-recommends \ + python3 openssl iproute2 iputils-ping curl ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /suite +COPY dstack-gateway /usr/local/bin/dstack-gateway +RUN chmod +x /usr/local/bin/dstack-gateway +COPY test_proxy.sh /suite/test_proxy.sh +COPY proxy /suite/proxy +RUN chmod +x /suite/test_proxy.sh + +ENV GATEWAY_BIN=/usr/local/bin/dstack-gateway +ENTRYPOINT ["/suite/test_proxy.sh"] diff --git a/dstack/gateway/test-run/proxy-e2e/README.md b/dstack/gateway/test-run/proxy-e2e/README.md new file mode 100644 index 000000000..5e8bbc4b3 --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/README.md @@ -0,0 +1,26 @@ + + +# Gateway proxy data-path suite + +Runs the gateway, the origin server and the probe client in one container. They +have to share a network namespace: `insecure_localhost_backend` resolves the +backend address to `127.0.0.1` as the gateway itself sees it. + +## Why the kTLS fallback arm runs in a second container + +One arm asserts that a gateway configured for kTLS on a kernel without the TLS +ULP falls back to userspace instead of truncating a gated transfer at the gate. +The suite used to produce that condition with `sudo rmmod tls`, which cannot +work in a container and which took the module away from the whole host. + +`notls-seccomp.json` makes `setsockopt(IPPROTO_TCP, TCP_ULP)` return +`ENOPROTOOPT` for one container instead, which is exactly what `probe_ktls()` +sees on a kernel without `CONFIG_TLS`. It touches nothing outside that +container and does not care what else on the machine is using TLS. + +A seccomp profile is fixed when a container is created, so the arm needs its +own container rather than a restart of the main one. diff --git a/dstack/gateway/test-run/proxy-e2e/docker-compose.yml b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml new file mode 100644 index 000000000..08e7645ce --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/docker-compose.yml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Two containers from one image. The suite restarts the gateway ~25 times with a +# different config each round; that loop stays inside a container rather than +# being handed to compose, which would turn every arm into a container create. + +# The gateway asks the guest agent for its own app id at startup. That is a unix +# socket, not a network hop, so sharing it through a volume leaves this suite's +# "everything in one network namespace" requirement untouched -- these services +# stay on network_mode: bridge and never join the fixture's network. +# +# Referenced rather than included, like the other two suites: one fixture +# project, one seed, one set of roots derived from it. +volumes: + dstack-socket: + external: true + name: dstack-attestation-socket + +x-suite: &suite + build: + context: . + dockerfile: Dockerfile + image: dstack-gateway-proxy-tests:local + cap_add: + - NET_ADMIN + volumes: + - dstack-socket:/var/run/dstack + # The suite's work directory, so the per-arm gateway logs outlive the + # container and CI has something to attach when an arm fails. + - ./run:/work + environment: + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + - WORK=/work + - KEEP_LOGS=1 + # The gateway reads /proc/net/tls_stat to report kTLS counters. In a container + # that MIB is per network namespace, so these numbers describe this run alone + # rather than whatever else on the machine happens to be using TLS. + network_mode: bridge + +services: + proxy-tests: + <<: *suite + container_name: gateway-proxy-tests + + # Same image, same suite, one arm. `setsockopt(IPPROTO_TCP, TCP_ULP)` is made + # to fail here so the gateway sees a kernel with no TLS ULP; a seccomp profile + # is fixed at creation, which is why this cannot just be another round inside + # the container above. + proxy-tests-notls: + <<: *suite + container_name: gateway-proxy-tests-notls + security_opt: + - seccomp=./notls-seccomp.json + # Repeats what the anchor sets: a service-level `environment` replaces the + # anchor's rather than merging with it, so omitting these here would leave + # this arm writing its logs somewhere the host never sees. + environment: + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + - WORK=/work + - KEEP_LOGS=1 + - GWTEST_ULP_UNAVAILABLE=1 diff --git a/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json new file mode 100644 index 000000000..80fb123ce --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json @@ -0,0 +1,14 @@ +{ + "defaultAction": "SCMP_ACT_ALLOW", + "syscalls": [ + { + "names": ["setsockopt"], + "action": "SCMP_ACT_ERRNO", + "errnoRet": 92, + "args": [ + {"index": 1, "value": 6, "op": "SCMP_CMP_EQ"}, + {"index": 2, "value": 31, "op": "SCMP_CMP_EQ"} + ] + } + ] +} diff --git a/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license new file mode 100644 index 000000000..7ed638e2a --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/notls-seccomp.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: © 2026 Phala Network + +SPDX-License-Identifier: Apache-2.0 diff --git a/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh new file mode 100755 index 000000000..228fd78d0 --- /dev/null +++ b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Host-side driver for the gateway proxy data-path suite. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_RUN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +SKIP_BUILD="" +case "${1:-}" in + --skip-build) SKIP_BUILD="--skip-build" ;; + down) docker compose -f "$SCRIPT_DIR/docker-compose.yml" down -v --remove-orphans; exit 0 ;; + -h|--help) echo "Usage: $0 [--skip-build] | down"; exit 0 ;; + "") ;; + *) echo "unknown option: $1" >&2; exit 1 ;; +esac + +# The suite needs the gateway binary and its own copies of the scripts in the +# build context. +"$TEST_RUN_DIR/build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD >/dev/null +cp "$TEST_RUN_DIR/test_proxy.sh" "$SCRIPT_DIR/test_proxy.sh" +rm -rf "$SCRIPT_DIR/proxy" && cp -r "$TEST_RUN_DIR/proxy" "$SCRIPT_DIR/proxy" + +# Two runs share one compose project, one set of container names and one work +# directory, so a second run tears down what the first is using. The damage does +# not look like a collision -- it looks like flaky tests, because state vanishes +# mid-run. Refuse to start instead. +# Locked on the directory, not on a file inside it. Deleting a lock FILE does +# not release the lock, it detaches it: the holder keeps its inode while the +# next run creates a fresh one and flocks that successfully. Three runs of the +# cluster suite overlapped exactly that way -- each wiping the others' state, +# with the failures reading as flaky sync rather than as a collision -- because +# a "clean up before rerunning" step had removed the lock file by name. +mkdir -p "$SCRIPT_DIR/run" +exec 9<"$SCRIPT_DIR/run" +if ! flock -n 9; then + echo "[ERROR] another run of this suite is already in progress ($SCRIPT_DIR/run)" >&2 + exit 1 +fi + +compose() { docker compose -f "$SCRIPT_DIR/docker-compose.yml" "$@"; } + +# The attestation fixture is a long-lived project shared with the other suites. +fixture_compose() { + docker compose -p dstack-fixture -f "$TEST_RUN_DIR/attestation/fixture.yml" "$@" +} + +# Both arms share one work directory; wipe it so a run never reads a previous +# run's logs. Two things this cannot do: +# +# - `rm -rf` it from the host. The suite container runs as root, so the certs +# and logs it leaves behind are root-owned and the host cannot remove them. +# The whole suite died here on its second local run, before a single +# assertion, with a wall of "Permission denied". CI never saw it because a +# fresh runner has no previous run to clean up. A throwaway container has +# the privileges the host lacks. +# +# - remove the directory ITSELF. `run` is what the lock above is held on, and +# replacing it with a fresh mkdir would detach that lock exactly the way a +# deleted lock file does -- the failure this suite's lock was just changed +# to prevent. Clear the contents and keep the inode. +mkdir -p "$SCRIPT_DIR/run" +docker run --rm -v "$SCRIPT_DIR/run:/r" alpine:latest \ + find /r -mindepth 1 -delete >/dev/null 2>&1 || true + +cleanup() { + compose down -v --remove-orphans >/dev/null 2>&1 || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi >/dev/null 2>&1 || true +} +trap cleanup EXIT + +compose build +fixture_compose build >/dev/null +echo "[INFO] starting the attestation fixture" >&2 +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) +fixture_compose up -d --wait >/dev/null + +echo "[INFO] running the proxy suite" >&2 +compose run --rm proxy-tests +MAIN_RC=$? + +echo "[INFO] running the no-TLS-ULP arm" >&2 +compose run --rm proxy-tests-notls +NOTLS_RC=$? + +[ "$MAIN_RC" -eq 0 ] && [ "$NOTLS_RC" -eq 0 ] diff --git a/dstack/gateway/test-run/proxy/gwconfig.py b/dstack/gateway/test-run/proxy/gwconfig.py index 46e488fd1..40b7d0f4f 100755 --- a/dstack/gateway/test-run/proxy/gwconfig.py +++ b/dstack/gateway/test-run/proxy/gwconfig.py @@ -58,7 +58,6 @@ def main(): set_ulimit = false [core.debug] insecure_localhost_backend = true -insecure_skip_attestation = true insecure_enable_debug_rpc = false [core.admin] enabled = true diff --git a/dstack/gateway/test-run/test_proxy.sh b/dstack/gateway/test-run/test_proxy.sh index 76d1ba929..010f4d6f5 100755 --- a/dstack/gateway/test-run/test_proxy.sh +++ b/dstack/gateway/test-run/test_proxy.sh @@ -12,8 +12,10 @@ # Complements `test_suite.sh`, which covers the control plane, WaveKV and the # handshake cache. This one is about bytes on the wire. # -# Requirements: python3, openssl, ip, sudo (once, to create the WireGuard-named -# link the gateway expects at startup). No root for the gateway itself. +# Runs inside the proxy-e2e container, which supplies python3, openssl and ip, +# and carries NET_ADMIN so the WireGuard-named link the gateway expects at +# startup can be created in the container's own namespace. Nothing here needs +# sudo, and nothing it creates outlives the container. # # ./test_proxy.sh # build and run everything # GATEWAY_BIN=/path/to/dstack-gateway ./test_proxy.sh @@ -81,7 +83,7 @@ cleanup() { stop_gateway [ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null if [ -n "${WG_CREATED:-}" ]; then - sudo ip link del "$WG_IFACE" 2>/dev/null + ip link del "$WG_IFACE" 2>/dev/null fi if [ $FAIL -eq 0 ] && [ -z "${KEEP_LOGS:-}" ]; then rm -rf "$WORK" @@ -131,16 +133,18 @@ setup() { # WireGuard device also makes `wg show` work, which the Status RPC needs; a # dummy link is enough for the data path, so fall back to one rather than # skipping every test on a host without the wireguard module. + # NET_ADMIN in the container's own namespace, so no sudo and nothing that + # outlives the run. if ip link show "$WG_IFACE" >/dev/null 2>&1; then WG_KIND=preexisting - elif sudo ip link add "$WG_IFACE" type wireguard 2>/dev/null; then + elif ip link add "$WG_IFACE" type wireguard 2>/dev/null; then WG_KIND=wireguard; WG_CREATED=1 - elif sudo ip link add "$WG_IFACE" type dummy 2>/dev/null; then + elif ip link add "$WG_IFACE" type dummy 2>/dev/null; then WG_KIND=dummy; WG_CREATED=1 else say "cannot create the link '$WG_IFACE' the gateway needs at startup"; exit 1 fi - [ -n "${WG_CREATED:-}" ] && sudo ip link set "$WG_IFACE" up + [ -n "${WG_CREATED:-}" ] && ip link set "$WG_IFACE" up say "link $WG_IFACE: $WG_KIND" CERT="$WORK/certs/cert.pem" KEY="$WORK/certs/key.pem" \ @@ -362,27 +366,30 @@ test_ktls_engages() { test_capability_fallbacks() { group "capability fallbacks announce themselves and keep serving" - # kTLS on a kernel without the TLS ULP must fall back, not truncate. Only - # exercised where the module can actually be taken away. - if [ "$(id -u)" = 0 ] || sudo -n true 2>/dev/null; then - if lsmod 2>/dev/null | grep -q "^tls " && sudo rmmod tls 2>/dev/null; then - echo "install tls /bin/false" | sudo tee /etc/modprobe.d/zz-gwtest-notls.conf >/dev/null - start_gateway "ktls-no-ulp" splice=immediate ktls=after:65536 - check "no TLS ULP / warns at startup" log_contains "kTLS is configured but unavailable" - check "no TLS ULP / small request intact" \ - probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1024 - # The regression this guards: a gated offload used to return HTTP 200 with - # the body cut off at exactly the gate. - check "no TLS ULP / large transfer not truncated" \ - probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 - stop_gateway - sudo rm -f /etc/modprobe.d/zz-gwtest-notls.conf - sudo modprobe tls 2>/dev/null - else - skip "kTLS fallback without the TLS ULP" "tls module not removable here" - fi + # kTLS on a kernel without the TLS ULP must fall back, not truncate. + # + # The condition used to be produced with `sudo rmmod tls`, which takes the + # module away from the entire host, needs passwordless sudo, and silently + # skips whenever anything else on the machine is using TLS. The suite runs in + # a container now, where a seccomp profile makes + # `setsockopt(IPPROTO_TCP, TCP_ULP)` return ENOPROTOOPT -- which is precisely + # what probe_ktls() sees on a kernel built without CONFIG_TLS, and affects + # nothing outside this container. + # + # A seccomp profile is fixed at container creation, so this arm gets its own + # container and the main run skips it. + if [ "${GWTEST_ULP_UNAVAILABLE:-0}" = 1 ]; then + start_gateway "ktls-no-ulp" splice=immediate ktls=after:65536 + check "no TLS ULP / warns at startup" log_contains "kTLS is configured but unavailable" + check "no TLS ULP / small request intact" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1024 + # The regression this guards: a gated offload used to return HTTP 200 with + # the body cut off at exactly the gate. + check "no TLS ULP / large transfer not truncated" \ + probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 1048576 + stop_gateway else - skip "kTLS fallback without the TLS ULP" "needs passwordless sudo" + skip "kTLS fallback without the TLS ULP" "runs in the notls container" fi start_gateway "rebalance-inert" splice=off ktls=off tpc=false rebalance=true \ From 0ecfe54d23a8a73bfb4b144266ba9ea417c58f4c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:48:23 -0700 Subject: [PATCH 07/27] test(gateway): verify peer quotes in the e2e suite The three e2e gateways built their own simulator and their own copy of the mock collateral service. They now reference the shared fixture's network and volumes, so there is one seed signing quotes and one set of roots derived from it, and the two cannot drift apart. `run-e2e.sh` owns the fixture's lifetime, which is why the gateways can no longer `depends_on` it. With that in place the gateways verify each other's quotes for real: `insecure_allow_external_trust_anchors` lets the anchor come from outside the vendor set, and every check in front of it runs the production path. Two fixes the run needed: `log_*` writes to stderr, so a helper whose stdout is captured no longer returns its log lines to the caller as data. Adding three ZT domains in a loop left two without certificates: the first domain on a fresh deployment creates the global ACME account and holds the shared lock while it does, and a domain added in that window is refused with "retry after it finishes" -- which nothing does. Retried here, as the error asks. The retry belongs in the product, not the harness; the suite is not the right place to fix a race it only reveals. --- .../test-run/e2e/configs/gateway-1.toml | 14 +++++ .../test-run/e2e/configs/gateway-2.toml | 14 +++++ .../test-run/e2e/configs/gateway-3.toml | 14 +++++ .../gateway/test-run/e2e/docker-compose.yml | 36 ++++++------ dstack/gateway/test-run/e2e/run-e2e.sh | 57 +++++++++++++++++-- dstack/gateway/test-run/e2e/test.sh | 54 +++++++++++------- 6 files changed, 148 insertions(+), 41 deletions(-) diff --git a/dstack/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml index b90efd4db..dec4e23ce 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-1.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-1.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml index c7bdd729f..975e0ab2e 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-2.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-2.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml index 0cb845126..a29b1de90 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-3.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-3.toml @@ -28,6 +28,20 @@ insecure_skip_attestation = false port = 9015 address = "0.0.0.0" +# Verify peer quotes against the development roots the mock collateral service +# derives from the simulator's seed. Verification itself runs the normal +# production path -- this only says the anchor may come from outside the vendor +# set, which is what makes attestation testable off TDX hardware. It does not +# switch any check off. +[core.attestation] +insecure_allow_external_trust_anchors = true + +[core.attestation.urls] +pccs = "http://mock-attestation:8088" + +[core.attestation.root_ca] +tdx = "/var/lib/attestation/tdx-root-ca.pem" + [core.sync] enabled = true interval = "5s" diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml index 7d90729ef..02d13425a 100644 --- a/dstack/gateway/test-run/e2e/docker-compose.yml +++ b/dstack/gateway/test-run/e2e/docker-compose.yml @@ -6,7 +6,14 @@ # Uses mock services: Pebble (ACME) + mock-cf-dns-api (Cloudflare DNS) # Uses a test-local dstack Guest Agent simulator for certificate and app identity flows. +# Same long-lived fixture project the cluster suite uses; run-e2e.sh brings it +# up. Referencing it rather than including it keeps one instance of the seed and +# the roots derived from it, and keeps this suite's network to itself. networks: + attestation: + external: true + name: dstack-attestation + certbot-test: driver: bridge ipam: @@ -14,23 +21,14 @@ networks: - subnet: 172.30.0.0/24 volumes: - pebble-certs: dstack-socket: + external: true + name: dstack-attestation-socket + attestation-roots: + external: true + name: dstack-attestation-roots services: - dstack-simulator: - build: - context: ../../../.. - dockerfile: dstack/gateway/test-run/e2e/Dockerfile.simulator - image: dstack-simulator:gateway-e2e - volumes: - - dstack-socket:/var/run/dstack - healthcheck: - test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] - interval: 1s - timeout: 1s - retries: 30 - # ==================== Mock Services ==================== # Mock Cloudflare DNS API. Built from the repo rather than pulled, and the @@ -100,6 +98,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-1 networks: + attestation: certbot-test: ipv4_address: 172.30.0.21 ports: @@ -110,14 +109,15 @@ services: volumes: - ./configs/gateway-1.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock + # The fixture is a separate project now; run-e2e.sh brings it up and waits + # for it before this one starts, so it cannot be a depends_on here. depends_on: - dstack-simulator: - condition: service_healthy mock-cf-dns-api: condition: service_healthy pebble: @@ -145,6 +145,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-2 networks: + attestation: certbot-test: ipv4_address: 172.30.0.22 ports: @@ -155,6 +156,7 @@ services: volumes: - ./configs/gateway-2.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: @@ -183,6 +185,7 @@ services: image: ${GATEWAY_IMAGE:-dstack-gateway:test} container_name: gateway-3 networks: + attestation: certbot-test: ipv4_address: 172.30.0.23 ports: @@ -193,6 +196,7 @@ services: volumes: - ./configs/gateway-3.toml:/etc/gateway/gateway.toml:ro - dstack-socket:/var/run/dstack + - attestation-roots:/var/lib/attestation:ro tmpfs: - /var/lib/gateway environment: diff --git a/dstack/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh index 7a7e0cccd..e60e6af34 100755 --- a/dstack/gateway/test-run/e2e/run-e2e.sh +++ b/dstack/gateway/test-run/e2e/run-e2e.sh @@ -10,6 +10,18 @@ set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +TEST_RUN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# The attestation fixture is its own long-lived project: one simulator, one +# collateral service, one seed. Both suites reference its network and volumes +# rather than each building a copy, so the seed that signs quotes and the seed +# that derives the verifying roots cannot drift apart. +# +# Defined here, above the EXIT trap that calls it -- a cleanup handler naming a +# function defined later fails on exactly the error paths it exists for. +fixture_compose() { + docker compose -p dstack-fixture -f "$TEST_RUN_DIR/attestation/fixture.yml" "$@" +} # Colors for output RED='\033[0;31m' @@ -72,10 +84,33 @@ done cd "$SCRIPT_DIR" # Cleanup function +# Two runs share one compose project, one set of container names and one work +# directory, so a second run tears down what the first is using. The damage does +# not look like a collision -- it looks like flaky tests, because state vanishes +# mid-run. Refuse to start instead. +# Locked on the directory, not on a file inside it. Deleting a lock FILE does +# not release the lock, it detaches it: the holder keeps its inode while the +# next run creates a fresh one and flocks that successfully. Three runs of the +# cluster suite overlapped exactly that way -- each wiping the others' state, +# with the failures reading as flaky sync rather than as a collision -- because +# a "clean up before rerunning" step had removed the lock file by name. +exec 9<"$SCRIPT_DIR" +if ! flock -n 9; then + log_error "another run of this suite is already in progress ($SCRIPT_DIR)" + exit 1 +fi + cleanup() { if ! $KEEP_RUNNING; then log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true + # Only if this run started it. The fixture is meant to outlive a single + # suite; tearing down one this run found already up is what made the next + # run fail with "network dstack-attestation declared as external, but could + # not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi fi } @@ -141,8 +176,19 @@ export GATEWAY_IMAGE=dstack-gateway:test # shared with the full-stack suite, so a stale copy from an older checkout # would be picked up silently -- and a mock that does not answer TCP fails # every challenge now that Pebble actually validates them. -log_info "Building the mock DNS/Cloudflare API..." -docker compose build mock-cf-dns-api +# Build every image rather than trusting whatever carries its tag. `image:` plus +# `build:` means compose reuses a local tag if one exists, and these tags are +# shared with other suites and other checkouts, so a stale copy would be picked +# up silently -- a mock that does not answer TCP fails every challenge now that +# Pebble validates them, and a stale simulator issues certificates from an older +# cert-client, which is the code path the cluster's app_id pinning depends on. +log_info "Building the mock services and the guest agent simulator..." +docker compose build +fixture_compose build + +log_info "Starting the attestation fixture..." +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) +fixture_compose up -d --wait docker compose up -d mock-cf-dns-api pebble log_info "Waiting for mock services to be healthy..." @@ -154,8 +200,11 @@ sleep 10 # Step 4: Run tests log_info "Running tests..." -docker compose run --rm test-runner -TEST_EXIT_CODE=$? +# `set -e` is on, so a plain call would abort the script the moment the suite +# fails and the report below would be unreachable -- the failure still exits +# non-zero, but silently. Capture the status instead of inheriting it. +TEST_EXIT_CODE=0 +docker compose run --rm test-runner || TEST_EXIT_CODE=$? # Step 5: Report result (cleanup handled by trap) if [ $TEST_EXIT_CODE -eq 0 ]; then diff --git a/dstack/gateway/test-run/e2e/test.sh b/dstack/gateway/test-run/e2e/test.sh index 9eecb98f7..66c224431 100755 --- a/dstack/gateway/test-run/e2e/test.sh +++ b/dstack/gateway/test-run/e2e/test.sh @@ -629,12 +629,24 @@ setup_certbot_config() { -d '{"domain": "'"${domain}"'", "port": 443}' > /dev/null \ || log_warn "AddZtDomain failed for $domain (may already exist)" + # The first domain on a fresh deployment has to create the global ACME + # account, and it holds the shared ACME lock while it does. A domain + # added inside that window is refused with "retry after it finishes" + # and nothing retries it promptly, so adding three in a tight loop + # leaves two without certificates. Retry here, as the error asks. log_info "Triggering renewal for: $domain" - curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ - -H "${ADMIN_AUTH_HEADER}" \ - -H "Content-Type: application/json" \ - -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null || \ - log_warn "Renewal request failed for $domain (may retry)" + local attempt=1 + while [ $attempt -le 10 ]; do + if curl -sf -X POST "${GATEWAY_ADMIN}/prpc/Admin.RenewZtDomainCert" \ + -H "${ADMIN_AUTH_HEADER}" \ + -H "Content-Type: application/json" \ + -d '{"domain": "'"${domain}"'", "force": true}' > /dev/null; then + break + fi + attempt=$((attempt + 1)) + sleep 3 + done + [ $attempt -le 10 ] || log_warn "Renewal never succeeded for $domain after 10 tries" done return 0 @@ -701,26 +713,26 @@ main() { # Phase 5: Certificate issuance log_phase 5 "Certificate issuance" - local first_domain first_sni first_proxy - first_domain=$(echo "$CERT_DOMAINS" | cut -d' ' -f1) - first_sni=$(get_test_sni "$first_domain") + local first_proxy first_proxy=$(echo "$GATEWAY_PROXIES" | cut -d' ' -f1) - log_info "Waiting for certificates (up to 120s)..." - local waited=0 - while [ $waited -lt 120 ]; do - if test_certificate_issued "$first_proxy" "$first_sni"; then - log_info "Certificate detected for $first_sni" - break - fi - sleep 5 - waited=$((waited + 5)) - log_info "Waiting... (${waited}s)" - done - - local sni wildcard + # Wait for each domain, not just the first. The orders run concurrently and + # finish in any order, so the first one landing says nothing about the rest + # -- asserting on all three the moment it does made the last domain a + # coin flip. + local sni waited for domain in $CERT_DOMAINS; do sni=$(get_test_sni "$domain") + log_info "Waiting for certificate: $domain (up to 120s)" + waited=0 + while [ $waited -lt 120 ]; do + if test_certificate_issued "$first_proxy" "$sni"; then + log_info "Certificate detected for $sni" + break + fi + sleep 5 + waited=$((waited + 5)) + done run_test "Certificate issued for $domain" \ "$(test_certificate_issued "$first_proxy" "$sni"; echo $?)" done From c6163ed6bf5c0af76fc5c76db59ae15d06a6c6d0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:48:33 -0700 Subject: [PATCH 08/27] ci: run the three gateway suites Only the proxy suite was in CI. The cluster suite and the e2e run were not, which is why the e2e rotted unnoticed for months and why the cluster suite still assumed a host it no longer got. Each workflow builds the gateway image, brings the attestation fixture up, runs its suite and tears everything down in a step that runs on failure too. Paths are scoped so a change to one component does not run all three. The proxy job's timeout goes from 30 to 45 minutes: it now builds the fixture images, which compile Rust from a cold docker cache on every run, and the cargo cache does not reach inside a docker build. --- .github/workflows/gateway-cluster-tests.yml | 110 +++++++++++++++++ .github/workflows/gateway-e2e-tests.yml | 124 ++++++++++++++++++++ .github/workflows/gateway-proxy-tests.yml | 44 +++++-- 3 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/gateway-cluster-tests.yml create mode 100644 .github/workflows/gateway-e2e-tests.yml diff --git a/.github/workflows/gateway-cluster-tests.yml b/.github/workflows/gateway-cluster-tests.yml new file mode 100644 index 000000000..bbe0f1f5a --- /dev/null +++ b/.github/workflows/gateway-cluster-tests.yml @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway cluster tests + +# WaveKV replication between gateway nodes: push and periodic sync, anti-entropy +# repair, bootstrap after losing a store, partition recovery, node identity, and +# the admin RPCs that gate registration. Unit tests cover the store; this runs +# three real gateways and stops, wipes and restarts them. +# +# The nodes authenticate each other for real, so the cluster mTLS path is +# exercised rather than switched off -- which is what the process-based suite +# this replaces could not do. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - '.github/workflows/gateway-cluster-tests.yml' + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-cluster: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # 28 tests, most of which restart nodes and then wait out a 5s sync interval. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-cluster-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-cluster-${{ runner.os }}- + + - name: Cluster suite + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh + + # The suite only materialises node logs for the two tests that assert on + # them, so the mounted directory is close to empty for any other failure. + # Pull them from the daemon instead, by compose service so the shared + # fixture's project-scoped names do not have to be guessed. + - name: Collect node logs on failure + if: failure() + working-directory: dstack/gateway/test-run/cluster + run: | + mkdir -p run/logs + for svc in $(docker compose config --services); do + docker compose logs --no-color "$svc" > "run/logs/$svc.log" 2>&1 || true + done + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + docker compose -p dstack-fixture -f ../attestation/fixture.yml logs \ + --no-color > "run/logs/fixture.log" 2>&1 || true + docker compose ps -a > run/logs/compose-ps.txt 2>&1 || true + + - name: Upload node logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-cluster-logs + path: dstack/gateway/test-run/cluster/run/logs/ + if-no-files-found: ignore + retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/cluster + run: ./run-cluster-tests.sh down diff --git a/.github/workflows/gateway-e2e-tests.yml b/.github/workflows/gateway-e2e-tests.yml new file mode 100644 index 000000000..f8f00f7a3 --- /dev/null +++ b/.github/workflows/gateway-e2e-tests.yml @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Gateway e2e tests + +# The gateway's certbot/ACME half only exists as a whole: certificate issuance, +# cluster sync of the result, and the dns-persist-01 flow each depend on the +# ones before. Unit tests cover the pieces; this stands up three gateways, a +# Pebble CA, a mock Cloudflare DNS API and a mock attestation collateral +# service, and asserts on what the cluster actually converges to. +# +# The suite verifies peer quotes for real: the gateways check each other against +# development trust anchors derived from the simulator's seed, so it covers the +# cluster mTLS path that a harness with the checks switched off cannot reach. +on: + push: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + pull_request: + branches: [ next, 'release/**' ] + paths: + - 'dstack/gateway/**' + - 'dstack/cert-client/**' + - 'dstack/certbot/**' + - 'dstack/ra-tls/**' + - 'dstack/guest-agent-simulator/**' + - 'dstack/crates/mock-attestation/**' + - 'tools/mock-cf-dns/**' + - '.github/workflows/gateway-e2e-tests.yml' + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +jobs: + gateway-e2e: + runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} + # A cold musl build dominates; the suite itself waits out real ACME orders + # and a 20s cluster-sync settle, so it is minutes rather than seconds. + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + + - name: Install Rust + uses: dtolnay/rust-toolchain@1.92.0 + with: + # run-e2e.sh builds a static gateway so the image can be alpine. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools + + # The gateways create real WireGuard interfaces inside their containers, + # so the host kernel has to offer the module. Record it: a run that failed + # because the runner image changed under us should say so plainly instead + # of looking like a gateway bug. + - name: Record kernel capabilities + run: | + echo "kernel: $(uname -r)" + sudo modprobe wireguard 2>&1 || echo "no wireguard module available" + echo "wireguard loaded: $(lsmod | grep -c '^wireguard ' || true)" + docker version --format 'docker: {{.Server.Version}}' + docker compose version + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + dstack/target + key: gateway-e2e-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} + restore-keys: gateway-e2e-${{ runner.os }}- + + # --keep-running so the containers survive for the log step below; the + # script's own EXIT trap would otherwise tear them down and leave nothing + # to collect from exactly the runs worth diagnosing. + - name: Gateway e2e suite + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh --keep-running + + - name: Collect container logs on failure + if: failure() + working-directory: dstack/gateway/test-run/e2e + run: | + # By compose service, not by container name: the shared attestation + # fixture deliberately does not pin one, so that more than one suite + # can be up at a time. + mkdir -p /tmp/gateway-e2e-logs + for svc in $(docker compose config --services); do + docker compose logs --no-color "$svc" > "/tmp/gateway-e2e-logs/$svc.log" 2>&1 || true + done + # The attestation fixture is a project of its own now, so it is not in + # this suite's service list and its logs have to be asked for + # separately -- they are where a quote-verification failure explains + # itself. + docker compose -p dstack-fixture -f ../attestation/fixture.yml logs \ + --no-color > "/tmp/gateway-e2e-logs/fixture.log" 2>&1 || true + docker compose ps -a > /tmp/gateway-e2e-logs/compose-ps.txt 2>&1 || true + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/e2e + run: ./run-e2e.sh down + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-e2e-logs + path: /tmp/gateway-e2e-logs/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/gateway-proxy-tests.yml b/.github/workflows/gateway-proxy-tests.yml index 48b56e528..b0df34257 100644 --- a/.github/workflows/gateway-proxy-tests.yml +++ b/.github/workflows/gateway-proxy-tests.yml @@ -8,6 +8,12 @@ name: Gateway proxy tests # `ktls`) whose behaviour depends on kernel capabilities and on a per-connection # gate. Unit tests cover the relay functions; this runs a real gateway process # and asserts on what actually reaches the wire. +# +# The suite runs in containers. One of them has a seccomp profile that makes +# `setsockopt(IPPROTO_TCP, TCP_ULP)` fail, which is how the no-TLS-ULP fallback +# is exercised now -- the suite used to `rmmod tls`, which needed passwordless +# sudo, took the module from the whole host, and skipped itself whenever +# anything else was using TLS. on: push: branches: [ next, 'release/**' ] @@ -32,13 +38,21 @@ jobs: proxy-integration: runs-on: ${{ vars.CI_RUNNER || 'ubuntu-latest' }} # Each of the ~25 arms restarts the gateway, and the idle-timeout arms wait - # out a real timeout, so this is minutes rather than seconds. - timeout-minutes: 30 + # out a real timeout. On top of that the suite now builds the shared + # attestation fixture, whose two images compile Rust from a cold docker cache + # on every run -- the cargo cache above does not reach inside a docker build. + timeout-minutes: 45 steps: - uses: actions/checkout@v5 - name: Install Rust uses: dtolnay/rust-toolchain@1.92.0 + with: + # The container image is built around a static binary. + targets: x86_64-unknown-linux-musl + + - name: Install musl toolchain + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools - name: Cache cargo uses: actions/cache@v4 @@ -50,10 +64,6 @@ jobs: key: gateway-proxy-${{ runner.os }}-${{ hashFiles('dstack/Cargo.lock') }} restore-keys: gateway-proxy-${{ runner.os }}- - - name: Build the gateway - working-directory: dstack - run: cargo build --release -p dstack-gateway - - name: Record kernel capabilities # The suite adapts to what the kernel offers, so the log needs to say # what it had: a run that skipped kTLS looks the same as one that @@ -66,16 +76,28 @@ jobs: | paste - - | sort -u || true - name: Proxy integration tests - working-directory: dstack/gateway/test-run - env: - GATEWAY_BIN: ${{ github.workspace }}/dstack/target/release/dstack-gateway - run: ./test_proxy.sh + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh + + - name: Collect container logs on failure + if: failure() + working-directory: dstack/gateway/test-run/proxy-e2e + run: | + mkdir -p run/logs + for svc in $(docker compose config --services); do + docker compose logs --no-color "$svc" > "run/logs/container-$svc.log" 2>&1 || true + done - name: Upload logs on failure if: failure() uses: actions/upload-artifact@v4 with: name: gateway-proxy-test-logs - path: /tmp/dstack-gw-proxy-test.*/logs/ + path: dstack/gateway/test-run/proxy-e2e/run/logs/ if-no-files-found: ignore retention-days: 7 + + - name: Tear down + if: always() + working-directory: dstack/gateway/test-run/proxy-e2e + run: ./run-proxy-tests.sh down From d722e60379dfa6e73ca4e7f5a3c9ba70d84d8aad Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 18:48:48 -0700 Subject: [PATCH 09/27] docs(gateway): rewrite the test-run guide for the compose suites TESTING.md described host processes, manual gateway startup and a certbot suite that no longer exists. It now describes what is there: three suites, what each covers, how to run one, and why they share a single attestation fixture. `.env.example` held Cloudflare credentials for `test_certbot.sh` and has nothing left to configure. The shellcheck exclude list loses the three deleted scripts. The replacements are not exempt; `e2e/run-e2e.sh` keeps its entry, unchanged. --- dstack/gateway/test-run/.env.example | 14 -- dstack/gateway/test-run/TESTING.md | 224 +++++++-------------------- prek.toml | 2 +- 3 files changed, 57 insertions(+), 183 deletions(-) delete mode 100644 dstack/gateway/test-run/.env.example diff --git a/dstack/gateway/test-run/.env.example b/dstack/gateway/test-run/.env.example deleted file mode 100644 index ff6571750..000000000 --- a/dstack/gateway/test-run/.env.example +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: © 2025 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Cloudflare API token with DNS edit permissions -# Required scopes: Zone.DNS (Edit), Zone.Zone (Read) -CF_API_TOKEN=your_cloudflare_api_token_here - -# Cloudflare Zone ID for your domain -CF_ZONE_ID=your_zone_id_here - -# Test domain (must be a wildcard domain managed by Cloudflare) -# Example: *.test.example.com -TEST_DOMAIN=*.test.example.com diff --git a/dstack/gateway/test-run/TESTING.md b/dstack/gateway/test-run/TESTING.md index 650297e63..96b23a95e 100644 --- a/dstack/gateway/test-run/TESTING.md +++ b/dstack/gateway/test-run/TESTING.md @@ -2,149 +2,86 @@ # # SPDX-License-Identifier: Apache-2.0 -# Gateway test plan +# Gateway integration tests -This document records the local checks used for the gateway handshake-cache -change. The goal is to verify three things: +Three suites, all under docker compose, all driven from the host. None of them +needs root on the machine running them, and none of them leaves anything behind. -1. the generic `cached-cell` crate behaves correctly; -2. existing gateway control-plane and WaveKV flows still work; -3. the proxy data path does not call blocking `wg show latest-handshakes` per - request. +| Suite | Covers | Run it | +| --- | --- | --- | +| `e2e/` | certbot, ACME, DNS-01 and `dns-persist-01` | `e2e/run-e2e.sh` | +| `cluster/` | WaveKV replication, node identity, partition recovery, admin RPCs | `cluster/run-cluster-tests.sh` | +| `proxy-e2e/` | the proxy data path: splice, kTLS, half-close, idle reaping | `proxy-e2e/run-proxy-tests.sh` | -## Prerequisites +Each has a `--skip-build` flag that reuses the `dstack-gateway` binary already +staged next to its compose file, and a `down` subcommand that tears everything +down. -Run on Linux with: +## Attestation is on -- Rust toolchain; -- `sudo`, `ip`, `wg` / `wireguard-tools`; -- `curl`, `openssl`, `python3`; -- `wrk` for the performance test. +No suite turns the checks off. The gateways obtain their +RPC certificates from the guest agent simulator, which signs quotes under trust +anchors derived from `attestation/tee-simulator.json`; the mock collateral +service reconstructs the matching public roots from the same seed, and the +gateways verify each other through the normal production path. -The integration script creates temporary WireGuard interfaces named -`wavekv-test1`, `wavekv-test2`, and `wavekv-test3`, so it needs root privileges. +That is the point of the arrangement rather than a detail of it: the cluster's +mTLS is what `cluster/` exists to exercise, and a suite that switched the checks +off would be testing the switch. -## Unit and build checks - -From the repository root: +## One fixture, one seed -```bash -cargo test -p cached-cell -cargo test --manifest-path gateway/Cargo.toml -cargo check --manifest-path gateway/Cargo.toml -cargo clippy -- \ - -D warnings \ - -D clippy::expect_used \ - -D clippy::unwrap_used \ - --allow unused_variables -``` +`attestation/fixture.yml` runs as its own long-lived compose project, started by +whichever suite you run. The suites reference its network and volumes rather +than each building a copy, so there is one simulator, one collateral service and +one seed. Two copies would drift apart, and every peer quote would then fail to +verify with nothing saying why. -Expected result: all commands pass. +## One project per test -## WaveKV / gateway integration test +The cluster suite gives each test a throwaway compose project of its own. Its +containers, its logs and its data directory are empty when it starts because +they are new -- not because something cleaned them up. -Build the gateway binary first: +That is the point rather than a detail. The version before it shared three +containers across all 28 tests and cleaned between them, which needed a wipe +helper, a time window on `docker logs`, and a lock to stop two runs from +clearing each other's state. Each of those three was the direct cause of a bug +during development, and each stops being possible here. -```bash -cargo build --release --manifest-path gateway/Cargo.toml -``` +## Unit and build checks -Then run the integration suite: +From `dstack/`: ```bash -cd gateway/test-run -sudo -E GATEWAY_BIN="$(pwd)/../../target/release/dstack-gateway" ./test_suite.sh -``` - -The suite starts real gateway processes and exercises: - -- CVM registration through `POST /prpc/RegisterCvm` on the debug service; -- admin RPCs such as `Admin.SetNodeUrl`, `Admin.SetNodeStatus`, and - `Admin.WaveKvStatus`; -- WaveKV persistent and ephemeral sync between gateway nodes; -- push propagation before the five-second periodic sync interval; -- periodic anti-entropy repair after a push is missed while a peer is offline; -- bootstrap recovery after a node loses its local WaveKV store while retaining - its node identity; -- convergence of divergent writes made on both sides of a partition; -- idempotence when opportunistic pushes overlap a periodic sync round; -- bootnode discovery retry, interrupted-sync recovery, and partial-cluster - bootstrap while another peer is unavailable; -- ephemeral-store convergence after a peer restart; -- node-ID conflict rejection followed by convergence under the replacement - node's fresh UUID; -- node restart, network partition recovery, periodic persistence, and node - up/down filtering. - -Expected result: - -```text -Tests passed: 28 +cargo test -p cached-cell +cargo test --manifest-path gateway/Cargo.toml +cargo clippy -- -D warnings -D clippy::expect_used -D clippy::unwrap_used \ + --allow unused_variables ``` -Important request paths covered by this suite: +## What runs in CI -| Path | Purpose | +| Workflow | Suite | | --- | --- | -| `POST /prpc/RegisterCvm` | Register a CVM, allocate a WireGuard IP, update gateway state. | -| `POST /prpc/Debug.Info` | Verify the debug service is available. | -| `POST /prpc/Debug.GetSyncData` | Inspect peer/node/instance data synced through WaveKV. | -| `POST /prpc/GetProxyState` | Compare in-memory proxy state with WaveKV state. | -| `POST /prpc/Admin.SetNodeUrl` | Register peer gateway URLs. | -| `POST /prpc/Admin.SetNodeStatus` | Mark nodes up/down and verify registration filtering. | -| `POST /prpc/Admin.WaveKvStatus` | Inspect WaveKV store status. | -| `POST /wavekv/sync/persistent` | Gateway-to-gateway persistent data sync. | -| `POST /wavekv/sync/ephemeral` | Gateway-to-gateway last-seen/handshake/connection sync. | -| `POST /wavekv/push/persistent` | Opportunistic persistent-state propagation. | -| `POST /wavekv/push/ephemeral` | Opportunistic ephemeral-state propagation. | - -## Real proxy data-path smoke test - -The integration suite above validates registration and sync, but it does not -open a client connection through the gateway proxy. For the proxy data path, use -this shape: - -1. Start one `dstack-gateway` with debug/admin enabled and `insecure_skip_attestation = true`. -2. Register a test CVM through the debug `RegisterCvm` RPC. -3. Bind a local HTTPS backend to the allocated CVM IP, for example - `10.0.51.2:23143`. -4. Serve a local DNS TXT response: - - ```text - _dstack-app-address.proxy-flow.local TXT "proxyflow:23143" - ``` - -5. Allow the backend port with `Admin.SetInstancePortPolicy` so the proxy data - path is not blocked by port-policy fail-close. -6. Send a request through the proxy: - - ```bash - curl -skf \ - --connect-to proxy-flow.local:13114:127.0.0.1:13114 \ - https://proxy-flow.local:13114/proxy-e2e - ``` - -Expected response from the backend: - -```text -proxy-e2e-ok path=/proxy-e2e -``` - -Expected gateway log shape: +| `.github/workflows/gateway-e2e-tests.yml` | `e2e/` | +| `.github/workflows/gateway-cluster-tests.yml` | `cluster/` | +| `.github/workflows/gateway-proxy-tests.yml` | `proxy-e2e/` | -```text -got sni: proxy-flow.local -target address is proxyflow:23143 -connecting to 10.0.51.2:23143 -connected to 10.0.51.2:23143 +```bash +gh pr checks --repo Dstack-TEE/dstack --watch=false ``` -This confirms the real data flow: +## The kTLS fallback arm -```text -client -> gateway proxy -> SNI parse -> DNS TXT lookup -> ProxyState selection -> backend TLS service -``` +One arm of the proxy suite asserts that a gateway configured for kTLS on a +kernel without the TLS ULP falls back to userspace instead of truncating a gated +transfer at the gate. It runs in its own container, because the condition is +produced by a seccomp profile -- `setsockopt(IPPROTO_TCP, TCP_ULP)` returns +`ENOPROTOOPT` -- and a seccomp profile is fixed when a container is created. +See `proxy-e2e/README.md` for why that replaced taking the module away from the +host with `rmmod`. ## Proxy performance / hot-path check The performance test uses the same real proxy data flow as the smoke test, with @@ -192,52 +129,3 @@ The important assertion is the call count: `wg show latest-handshakes` is only used by startup/preload and the periodic refresh task, not once per proxied request. -## PR CI - -Check GitHub Actions before merging: - -```bash -gh pr checks --repo Dstack-TEE/dstack --watch=false -``` - -Expected result: all required checks pass, including `gateway`, `rust-checks`, -`prek`, `reuse-lint`, and CodeQL. - -## Proxy data-path integration tests - -`test_proxy.sh` runs a real gateway process and asserts on what reaches the -wire, across every combination of the two gated optimisations and both proxy -paths. It runs in CI (`.github/workflows/gateway-proxy-tests.yml`); unlike -`test_suite.sh` it needs no root for the gateway itself, only one `sudo ip link -add` for the link the gateway expects at startup. - -```bash -cd gateway/test-run -./test_proxy.sh # builds the gateway if needed -GATEWAY_BIN=../../target/release/dstack-gateway ./test_proxy.sh -BASE_PORT=39000 ./test_proxy.sh # if the default range is busy -KEEP_LOGS=1 ./test_proxy.sh # keep the work dir on success -``` - -What it covers: - -| group | asserts | -|---|---| -| data path | payloads survive byte-for-byte on both paths, under and over each gate, and under concurrency | -| close | the app closing reaches the client as an orderly TLS shutdown, including after kTLS offload | -| `timeouts.idle` | the idle watchdog reaps on every relay path -- buffered, spliced, kTLS -- and `data_timeout_enabled = false` still opts out | -| kTLS engagement | offload happens, and only once the gate fires; no TLS decrypt errors | -| capability fallbacks | a kernel without the TLS ULP warns and keeps serving untruncated; an inert `connection_rebalance` warns | -| runtime modes | every `thread_per_core` / `connection_rebalance` combination serves traffic | -| half-close | a client that finishes its request still gets the reply, on both paths and every arm | -| Status RPC | the accel counters reflect the traffic that ran | - -The suite adapts to the host: groups that need a capability the kernel or the -privileges do not provide are reported as `SKIP` with the reason, never silently -passed. - -Half-close needs a TLS client that can send `close_notify` without waiting for -the peer's, which `ssl.SSLSocket` cannot express: its only shutdown is -bidirectional, and dropping to `shutdown(SHUT_WR)` sends a bare FIN, which -mid-TLS is a truncation the peer is right to reject. `proxy/tlsclient.py` -drives the TLS state machine over memory BIOs to do it properly. diff --git a/prek.toml b/prek.toml index bb2438336..acebd0f3f 100644 --- a/prek.toml +++ b/prek.toml @@ -65,7 +65,7 @@ hooks = [ # The imported Yocto backend retains its upstream shell style. New common # OS scripts remain covered by this hook. The explicit legacy list records # pre-existing shellcheck debt exposed when the files moved into dstack/. - { id = "shellcheck", exclude = '^(os/yocto/.*|tools/(dev-stack\.sh|vm-runner/.*)|dstack/(cargo-check-all\.sh|gateway/test-run/(cluster\.sh|e2e/run-e2e\.sh|test_certbot\.sh)|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, + { id = "shellcheck", exclude = '^(os/yocto/.*|tools/(dev-stack\.sh|vm-runner/.*)|dstack/(cargo-check-all\.sh|gateway/test-run/e2e/run-e2e\.sh|guest-agent-simulator/install-systemd\.sh|kms/auth-eth/run-tests\.sh|scripts/(config-fw\.sh|setup-bridge\.sh)|supervisor/tests/(test-cli\.sh|test\.sh)|test-scripts/(get-app-key\.sh|inspect-cert\.sh)|verifier/test\.sh|vmm/(src/setup-user\.sh|src/tests/test-deployment\.sh|ui/scripts/build_proto\.sh|venv\.sh)))$' }, ] # --- Conventional commits (used by cliff.toml for changelog) --- From 57b281ad0927be1826bc92bd15a5e8f10c1a65e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 20:41:18 -0700 Subject: [PATCH 10/27] test(ra-tls): add a shared test PKI instead of hand-rolling certs per suite Every crate that needs a certificate to drive a TLS test built one out of raw `rcgen`: generate a key, set `IsCa::Ca(BasicConstraints::Unconstrained)`, self-sign, write three PEM files. The same twenty lines are in the gateway's sync tests, its HTTPS client tests, and `ra-rpc`'s client-auth tests, each slightly different, and none of them found `CertRequest::ca_level()` -- which has done the CA half since it was added. `wavekv_sync`'s copy shows what that costs. Because it went around `CertRequest`, it could not use `.app_id()` either, so it hand-encoded the DER OCTET STRING for the extension: debug_assert!(TEST_APP_ID.len() < 128); let mut app_id_der = vec![0x04, TEST_APP_ID.len() as u8]; Three bytes of ASN.1 header, a `debug_assert` pinning the short-form length, and a comment explaining why -- to set a field the builder already sets. Added `ra_tls::test_pki` behind an off-by-default `test-pki` feature: - `TestCa` -- a self-signed CA, `ca_level(0)` since a test CA has no reason to mint intermediates. - `TestCert` -- a leaf to mint, self-signed or CA-signed, with `.app_id()`, alt names and usage flags. `TestCert::localhost()` is the shape local TLS tests want: valid for `127.0.0.1`, usable at both ends of an mTLS connection. - `write_mtls_pki` -- CA plus one leaf, written as the `node.crt`/`node.key`/ `ca.crt` trio a client config points at. Returns both, so a test needing a second peer can sign one under the same CA. Nothing here needs a TEE: the app_id extension is an ordinary X.509 extension and a peer check that compares app ids never looks at a quote. Material that does carry attestation still goes through `generate_ra_cert_with_app_id`. The feature is off by default and reaches the gateway as a dev-dependency on the crate its normal dependency already names, so it is enabled for tests and not for the binary. `cargo tree --no-dev-dependencies` shows `ra-tls` with `default` only; with dev-dependencies it shows `test-pki`. Migrated `wavekv_sync`'s three helpers: -61/+18 lines, hand-encoded DER gone. `https_client`'s copies are left for the branch that already rewrites them. ra-tls 35 passed (31 + 4 new), dstack-gateway 308 passed, clippy and fmt clean, and `cargo build -p dstack-gateway` still builds without the feature. --- dstack/Cargo.lock | 1 + dstack/gateway/Cargo.toml | 3 + dstack/gateway/src/web_routes/wavekv_sync.rs | 79 ++--- dstack/ra-tls/Cargo.toml | 7 + dstack/ra-tls/src/lib.rs | 2 + dstack/ra-tls/src/test_pki.rs | 292 +++++++++++++++++++ 6 files changed, 323 insertions(+), 61 deletions(-) create mode 100644 dstack/ra-tls/src/test_pki.rs diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 0ac7ac434..46f6b6727 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -5818,6 +5818,7 @@ dependencies = [ "sha2 0.10.9", "sha3", "tdx-attest", + "tempfile", "tokio", "tpm-qvl", "tpm-types", diff --git a/dstack/gateway/Cargo.toml b/dstack/gateway/Cargo.toml index 0058ec068..e682bb690 100644 --- a/dstack/gateway/Cargo.toml +++ b/dstack/gateway/Cargo.toml @@ -74,6 +74,9 @@ socket2.workspace = true [dev-dependencies] insta.workspace = true +# The test-only CA/leaf helpers. A dev-dependency on the same crate the normal +# dependency points at, so the feature is enabled for tests and not for the binary. +ra-tls = { workspace = true, features = ["test-pki"] } rmpv.workspace = true tempfile.workspace = true wavekv-v1 = { package = "wavekv", version = "=1.0.0" } diff --git a/dstack/gateway/src/web_routes/wavekv_sync.rs b/dstack/gateway/src/web_routes/wavekv_sync.rs index 066795b9e..3f400031b 100644 --- a/dstack/gateway/src/web_routes/wavekv_sync.rs +++ b/dstack/gateway/src/web_routes/wavekv_sync.rs @@ -264,49 +264,22 @@ mod tests { /// A self-signed CA plus a leaf it signs. `HttpSyncNetwork::new` loads all three /// from disk to build its rustls client config, and the root store only accepts a /// trust anchor with `CA:TRUE` — so a lone self-signed leaf is not enough. + /// + /// The leaf carries an app_id because no test here turns the peer check off, so a + /// certificate without one is refused before any of them reach what they are about. + /// It is also what a real peer's certificate carries. fn write_tls_material(dir: &std::path::Path) -> TlsConfig { - use ra_tls::rcgen::{BasicConstraints, CertificateParams, IsCa, KeyPair}; - - let ca_key = KeyPair::generate().expect("ca key"); - let mut ca_params = CertificateParams::new(vec![]).expect("ca params"); - ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); - let ca_cert = ca_params.self_signed(&ca_key).expect("ca cert"); - - let leaf_key = KeyPair::generate().expect("leaf key"); - let mut leaf_params = - CertificateParams::new(vec!["127.0.0.1".to_string()]).expect("leaf params"); - // No test here turns the peer check off, so a certificate without an app_id - // is refused before any of them reach what they are about. Stamping one - // here is also what a real peer's certificate carries. - // A DER OCTET STRING, which is what `ra_tls` writes into this extension. - // Hand-encoded rather than pulling in an ASN.1 crate for three bytes of - // header; the short-form length is valid because the payload is under 128 - // bytes, which a debug_assert pins. - debug_assert!(TEST_APP_ID.len() < 128); - let mut app_id_der = vec![0x04, TEST_APP_ID.len() as u8]; - app_id_der.extend_from_slice(TEST_APP_ID); - leaf_params - .custom_extensions - .push(ra_tls::rcgen::CustomExtension::from_oid_content( - ra_tls::oids::PHALA_RATLS_APP_ID, - app_id_der, - )); - let leaf_cert = leaf_params - .signed_by(&leaf_key, &ca_cert, &ca_key) - .expect("leaf cert"); - - let cert_path = dir.join("node.crt"); - let key_path = dir.join("node.key"); - let ca_path = dir.join("ca.crt"); - std::fs::write(&cert_path, leaf_cert.pem()).expect("write cert"); - std::fs::write(&key_path, leaf_key.serialize_pem()).expect("write key"); - std::fs::write(&ca_path, ca_cert.pem()).expect("write ca"); + let pki = ra_tls::test_pki::write_mtls_pki( + dir, + ra_tls::test_pki::TestCert::localhost().app_id(TEST_APP_ID), + ) + .expect("write test PKI"); TlsConfig { - certs: cert_path.to_string_lossy().into_owned(), - key: key_path.to_string_lossy().into_owned(), + certs: pki.cert_path.to_string_lossy().into_owned(), + key: pki.key_path.to_string_lossy().into_owned(), mutual: MutualConfig { - ca_certs: ca_path.to_string_lossy().into_owned(), + ca_certs: pki.ca_cert_path.to_string_lossy().into_owned(), }, } } @@ -408,33 +381,17 @@ mod tests { /// `CertRequest` adds unconditionally, and the check under test never looks at a /// quote — it reads two extensions and compares bytes. fn cert_with_app_id(app_id: &[u8]) -> Vec { - use ra_tls::cert::CertRequest; - use ra_tls::rcgen::KeyPair; - - let key = KeyPair::generate().expect("key"); - let cert = CertRequest::builder() - .key(&key) - .subject("peer.test") + ra_tls::test_pki::TestCert::new("peer.test") .app_id(app_id) - .build() - .self_signed() - .expect("self-signed cert"); - cert.der().to_vec() + .self_signed_der() + .expect("self-signed cert") } /// A certificate with no app identity at all. fn cert_without_app_id() -> Vec { - use ra_tls::cert::CertRequest; - use ra_tls::rcgen::KeyPair; - - let key = KeyPair::generate().expect("key"); - let cert = CertRequest::builder() - .key(&key) - .subject("peer.test") - .build() - .self_signed() - .expect("self-signed cert"); - cert.der().to_vec() + ra_tls::test_pki::TestCert::new("peer.test") + .self_signed_der() + .expect("self-signed cert") } fn authorize(der: &[u8], my_app_id: Option<&[u8]>) -> Result<(), Status> { diff --git a/dstack/ra-tls/Cargo.toml b/dstack/ra-tls/Cargo.toml index 2530e695b..236f85ebf 100644 --- a/dstack/ra-tls/Cargo.toml +++ b/dstack/ra-tls/Cargo.toml @@ -49,7 +49,14 @@ rmp-serde.workspace = true [features] simulator = ["dstack-attest/simulator"] quote = ["dstack-attest/quote"] +# A self-signed CA and the leaves it signs, for tests that need a certificate but no +# TEE. Off by default: the keys it mints are test material and must not be reachable +# from a production binary. +test-pki = [] [dev-dependencies] ed25519-dalek.workspace = true +# `test_pki`'s own tests write PEMs to a scratch dir; the module itself takes a +# caller-supplied path and does not depend on this. +tempfile.workspace = true tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/dstack/ra-tls/src/lib.rs b/dstack/ra-tls/src/lib.rs index 36dc07585..8ea8adf48 100644 --- a/dstack/ra-tls/src/lib.rs +++ b/dstack/ra-tls/src/lib.rs @@ -13,4 +13,6 @@ pub mod api_v1; pub mod cert; pub mod kdf; pub mod oids; +#[cfg(feature = "test-pki")] +pub mod test_pki; pub mod traits; diff --git a/dstack/ra-tls/src/test_pki.rs b/dstack/ra-tls/src/test_pki.rs new file mode 100644 index 000000000..87f5517c9 --- /dev/null +++ b/dstack/ra-tls/src/test_pki.rs @@ -0,0 +1,292 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Test-only PKI: a self-signed CA and the leaves it signs. +//! +//! Every crate that needs a certificate to drive a TLS test used to build one out of +//! raw `rcgen` -- generate a key, set `IsCa::Ca(BasicConstraints::Unconstrained)`, +//! self-sign, write three PEM files -- which is how the same twenty lines ended up in +//! `gateway`'s sync tests, its HTTPS client tests, and `ra-rpc`'s client-auth tests, +//! each subtly different. +//! +//! Nothing here needs a TEE. `PHALA_RATLS_APP_ID` is an ordinary X.509 extension that +//! [`CertRequest`] writes unconditionally, and a peer check that compares app ids never +//! looks at a quote. For material that *does* carry attestation, use +//! [`generate_ra_cert_with_app_id`](crate::cert::generate_ra_cert_with_app_id), which +//! needs the `quote` feature and a real or simulated platform. +//! +//! Private keys produced here are test material and must never reach a production +//! image, which is why this module is behind the off-by-default `test-pki` feature. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use rcgen::{Certificate, KeyPair}; + +use crate::cert::CertRequest; + +/// A certificate and the key that signs for it. +pub struct TestCertKey { + /// The certificate. + pub cert: Certificate, + /// Its private key. + pub key: KeyPair, +} + +impl TestCertKey { + /// The certificate, PEM-encoded. + pub fn cert_pem(&self) -> String { + self.cert.pem() + } + + /// The private key, PEM-encoded. + pub fn key_pem(&self) -> String { + self.key.serialize_pem() + } + + /// The certificate, DER-encoded. + pub fn cert_der(&self) -> Vec { + self.cert.der().to_vec() + } + + /// The private key, DER-encoded. + pub fn key_der(&self) -> Vec { + self.key.serialize_der() + } +} + +/// A self-signed CA that exists to sign test leaves. +/// +/// `ca_level(0)` rather than an unconstrained CA: a test CA has no reason to be allowed +/// to mint intermediates, and a root store that only accepts `CA:TRUE` is satisfied +/// either way. +pub struct TestCa(TestCertKey); + +impl TestCa { + /// Mint a new CA. + pub fn new() -> Result { + Self::named("dstack Test CA") + } + + /// Mint a new CA with a chosen subject. + pub fn named(subject: &str) -> Result { + let key = KeyPair::generate().context("failed to generate CA key")?; + let cert = CertRequest::builder() + .subject(subject) + .key(&key) + .ca_level(0) + .build() + .self_signed() + .context("failed to self-sign CA")?; + Ok(Self(TestCertKey { cert, key })) + } + + /// The CA certificate and key. + pub fn cert_key(&self) -> &TestCertKey { + &self.0 + } + + /// The CA certificate, PEM-encoded -- what a client's root store loads. + pub fn cert_pem(&self) -> String { + self.0.cert_pem() + } +} + +/// A leaf to mint, self-signed or signed by a [`TestCa`]. +#[derive(Default)] +pub struct TestCert { + subject: String, + alt_names: Vec, + app_id: Option>, + server_auth: bool, + client_auth: bool, +} + +impl TestCert { + /// A leaf with the given subject and no extensions beyond the defaults. + pub fn new(subject: &str) -> Self { + Self { + subject: subject.to_string(), + ..Default::default() + } + } + + /// A leaf valid for `127.0.0.1`, usable for both ends of an mTLS connection. + /// + /// The shape almost every local TLS test wants: Rocket and `rustls` both check the + /// SAN against the address dialled, and a test that reuses one leaf for the server + /// and the client needs both usages. + pub fn localhost() -> Self { + Self::new("localhost") + .alt_name("127.0.0.1") + .server_auth(true) + .client_auth(true) + } + + /// Add a subject alternative name. + pub fn alt_name(mut self, name: &str) -> Self { + self.alt_names.push(name.to_string()); + self + } + + /// Stamp `PHALA_RATLS_APP_ID`, which is what a peer identity check reads. + pub fn app_id(mut self, app_id: &[u8]) -> Self { + self.app_id = Some(app_id.to_vec()); + self + } + + /// Mark the leaf usable for server authentication. + pub fn server_auth(mut self, yes: bool) -> Self { + self.server_auth = yes; + self + } + + /// Mark the leaf usable for client authentication. + pub fn client_auth(mut self, yes: bool) -> Self { + self.client_auth = yes; + self + } + + fn request<'a>(&'a self, key: &'a KeyPair) -> crate::cert::CertRequest<'a, KeyPair> { + CertRequest::builder() + .subject(&self.subject) + .key(key) + .alt_names(&self.alt_names) + .maybe_app_id(self.app_id.as_deref()) + .usage_server_auth(self.server_auth) + .usage_client_auth(self.client_auth) + .build() + } + + /// Mint the leaf, signed by itself. + pub fn self_signed(self) -> Result { + let key = KeyPair::generate().context("failed to generate leaf key")?; + let cert = self + .request(&key) + .self_signed() + .context("failed to self-sign leaf")?; + Ok(TestCertKey { cert, key }) + } + + /// Mint the leaf, signed by `ca`. + pub fn signed_by(self, ca: &TestCa) -> Result { + let key = KeyPair::generate().context("failed to generate leaf key")?; + let ca = ca.cert_key(); + let cert = self + .request(&key) + .signed_by(&ca.cert, &ca.key) + .context("failed to sign leaf")?; + Ok(TestCertKey { cert, key }) + } + + /// Mint the leaf DER only, signed by itself. + /// + /// For a check that parses a certificate and reads an extension, where the key is + /// never used to complete a handshake. + pub fn self_signed_der(self) -> Result> { + Ok(self.self_signed()?.cert_der()) + } +} + +/// Where [`write_mtls_pki`] put the PEM files. +pub struct TestPkiFiles { + /// The leaf certificate, at `/node.crt`. + pub cert_path: PathBuf, + /// The leaf private key, at `/node.key`. + pub key_path: PathBuf, + /// The CA certificate, at `/ca.crt`. + pub ca_cert_path: PathBuf, + /// The leaf, for a test that also needs to serve with it. + pub leaf: TestCertKey, + /// The CA, for a test that signs a second leaf to play the other end. + pub ca: TestCa, +} + +/// Mint a CA and one leaf it signed, and write all three PEM files under `dir`. +/// +/// The layout clients expect: a config that names a cert, a key and a CA bundle can +/// point straight at the three paths returned. Both the leaf and the CA come back, so a +/// test that needs a second peer -- a different app id, an expired leaf -- can sign one +/// under the same CA rather than starting over. +pub fn write_mtls_pki(dir: &Path, leaf: TestCert) -> Result { + let ca = TestCa::new()?; + let leaf = leaf.signed_by(&ca)?; + + let cert_path = dir.join("node.crt"); + let key_path = dir.join("node.key"); + let ca_cert_path = dir.join("ca.crt"); + + std::fs::write(&cert_path, leaf.cert_pem()).context("failed to write leaf cert")?; + std::fs::write(&key_path, leaf.key_pem()).context("failed to write leaf key")?; + std::fs::write(&ca_cert_path, ca.cert_pem()).context("failed to write CA cert")?; + + Ok(TestPkiFiles { + cert_path, + key_path, + ca_cert_path, + leaf, + ca, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::CertExt; + + #[test] + fn a_leaf_carries_the_app_id_it_was_stamped_with() { + let leaf = TestCert::localhost() + .app_id(b"an-app-id") + .self_signed() + .expect("leaf"); + assert_eq!( + leaf.cert.get_app_id().expect("read app id"), + Some(b"an-app-id".to_vec()) + ); + } + + #[test] + fn a_leaf_without_an_app_id_reads_back_as_absent() { + let leaf = TestCert::localhost().self_signed().expect("leaf"); + assert_eq!(leaf.cert.get_app_id().expect("read app id"), None); + } + + /// A root store built from the CA only accepts a trust anchor with `CA:TRUE`, so a + /// CA that did not come back as one would fail every handshake rather than an + /// assertion here. + #[test] + fn the_ca_is_a_ca_and_signs_leaves_that_chain_to_it() { + use x509_parser::prelude::FromDer; + + let ca = TestCa::new().expect("ca"); + let leaf = TestCert::localhost().signed_by(&ca).expect("leaf"); + + let ca_der = ca.cert_key().cert_der(); + let (_, parsed_ca) = + x509_parser::certificate::X509Certificate::from_der(&ca_der).expect("parse ca"); + assert!(parsed_ca.is_ca(), "the test CA must be usable as one"); + + let leaf_der = leaf.cert_der(); + let (_, parsed_leaf) = + x509_parser::certificate::X509Certificate::from_der(&leaf_der).expect("parse leaf"); + assert_eq!( + parsed_leaf.issuer(), + parsed_ca.subject(), + "a leaf signed by the CA must name it as issuer" + ); + } + + #[test] + fn the_three_pem_files_land_where_a_client_config_points() { + let dir = tempfile::tempdir().expect("tempdir"); + let pki = write_mtls_pki(dir.path(), TestCert::localhost().app_id(b"an-app-id")) + .expect("write pki"); + + for path in [&pki.cert_path, &pki.key_path, &pki.ca_cert_path] { + let pem = std::fs::read_to_string(path).expect("read pem"); + assert!(pem.contains("-----BEGIN"), "{path:?} is not PEM"); + } + } +} From a166fd2d8a8f957e6e567fc3aaa5d83ac78be9b6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:27:54 -0700 Subject: [PATCH 11/27] test(gateway): make a cluster rerun start from nothing Node state is a bind mount, so `compose down -v` does not remove it, and the gateway writes its certificates 0600 as root, so the host user cannot either. Test names are fixed. Every run after the first therefore resumed each test on the store the same test left behind -- which is the opposite of what `docker-compose.yml` and `tests.sh` say happens. Measured, not reasoned about: with `setup_peers` stubbed so nodes never learn about each other, `test_cross_node_data_sync` passed on a second run and the suite exited 0. Node 2 found its instance already in the snapshot from the run before. With the wipe in place the same mutation fails with `kv2=0`. The same shape reaches `test_partial_cluster_bootstrap`, `test_delayed_bootnode_recovery` and the `keys_after_write > 0` guard in `test_persistence`. The clear-out runs in a throwaway root container, the way the proxy suite already does it, and removes contents only: `run` is what the suite's lock is held on, and replacing the directory would detach that lock exactly the way a deleted lock file does. Teardown is fixed alongside, because it is what leaves the state behind. `compose down` with CURRENT_TEST unset addresses `cluster-suite`, which no test ever uses -- the runner sets CURRENT_TEST before the first one -- so the `down` subcommand tore down nothing, and CI's log collection ran against a `cluster` project that has no containers. Both now ask the daemon which projects came from this compose file, so a killed run's leftovers are found whatever they are called. --- .github/workflows/gateway-cluster-tests.yml | 15 ++++-- dstack/gateway/test-run/cluster/lib.sh | 53 +++++++++++++++++++ .../test-run/cluster/run-cluster-tests.sh | 13 +++-- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-cluster-tests.yml b/.github/workflows/gateway-cluster-tests.yml index bbe0f1f5a..b354df60b 100644 --- a/.github/workflows/gateway-cluster-tests.yml +++ b/.github/workflows/gateway-cluster-tests.yml @@ -84,9 +84,18 @@ jobs: working-directory: dstack/gateway/test-run/cluster run: | mkdir -p run/logs - for svc in $(docker compose config --services); do - docker compose logs --no-color "$svc" > "run/logs/$svc.log" 2>&1 || true - done + # By project, discovered from the daemon. A bare `docker compose` here + # addresses the project named after this directory (`cluster`), which + # no test ever uses -- every test runs under `cluster-`, so + # the loop that used `compose config --services` collected nothing. + docker ps -a --filter 'label=com.docker.compose.project' \ + --format '{{.Label "com.docker.compose.project"}}' \ + | grep -E '^cluster-' | sort -u | while read -r project; do + docker compose -p "$project" -f docker-compose.yml logs --no-color \ + > "run/logs/project-$project.log" 2>&1 || true + done + # The suite's own per-test dumps are root-owned inside the bind mount. + docker run --rm -v "$PWD/run:/r" alpine:latest chmod -R a+rX /r || true # The attestation fixture is a project of its own now, so it is not in # this suite's service list and its logs have to be asked for # separately -- they are where a quote-verification failure explains diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh index 69005403b..c0a4d0bd4 100644 --- a/dstack/gateway/test-run/cluster/lib.sh +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -178,6 +178,59 @@ fixture_compose() { -f "$SCRIPT_DIR/../attestation/fixture.yml" "$@" } +# Every project this suite may have created, asked of the daemon rather than +# guessed. +# +# `compose down` with CURRENT_TEST unset addresses `cluster-suite`, which no +# test ever uses -- the runner sets CURRENT_TEST before the first one -- so the +# `down` subcommand tore down nothing and a killed run leaked one project per +# test, containers still attached to the shared attestation network. Label +# lookup finds them whatever they are called and whatever went wrong. +cluster_projects() { + # Matched on the compose file's absolute path, not on a `cluster-*` name + # pattern. The name is this suite's convention, not something the daemon + # guarantees, and a pattern wide enough to catch every per-test project is + # also wide enough to tear down an unrelated `cluster-…` project that + # happens to be on the same machine. `config_files` is set by compose to the + # file the container came from, so this selects exactly this checkout's + # containers -- another worktree running the same suite is left alone. + docker ps -a \ + --filter "label=com.docker.compose.project.config_files=$SCRIPT_DIR/docker-compose.yml" \ + --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \ + | sort -u +} + +down_all() { + local project + while read -r project; do + [ -n "$project" ] || continue + docker compose -p "$project" -f "$SCRIPT_DIR/docker-compose.yml" \ + down -v --remove-orphans >/dev/null 2>&1 || true + done < <(cluster_projects) +} + +# Clear the run tree's contents, from inside a container. +# +# The gateway runs as root and writes its certificates 0600 (see +# `safe_write_with_mode` in gateway/src/main.rs), so the host user cannot delete +# what a previous run left. Node state is a bind mount, which `compose down -v` +# does not touch either -- so without this every test resumed from the previous +# run's store. Test names are fixed, so the stale directory is always the one +# the test is about to use: `test_cross_node_data_sync` found its instance +# already replicated and passed with sync entirely broken, and +# `test_partial_cluster_bootstrap` never had to bootstrap. Only the first run on +# a fresh checkout -- which is what CI does, and what this was verified on -- +# behaved as the comments describe. +# +# Contents only. `run` is what the suite's lock is held on, and replacing the +# directory would detach that lock exactly the way a deleted lock file does. +wipe_run_tree() { + mkdir -p "$RUN_DIR" + docker run --rm -v "$RUN_DIR:/r" alpine:latest \ + find /r -mindepth 1 -delete >/dev/null 2>&1 \ + || log_warn "could not clear $RUN_DIR; a stale store may make assertions vacuous" +} + # Discard whatever the previous test used and start this one from nothing. new_test_project() { compose down -v --remove-orphans >/dev/null 2>&1 || true diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh index 563f91723..1cfd70224 100755 --- a/dstack/gateway/test-run/cluster/run-cluster-tests.sh +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -24,7 +24,12 @@ while [[ $# -gt 0 ]]; do --keep-running) KEEP_RUNNING=true; shift ;; --only) ONLY="$2"; shift 2 ;; down) - compose down -v --remove-orphans 2>/dev/null || true + # Every per-test project, not just the one CURRENT_TEST happens to + # name, plus the shared fixture -- otherwise CI's teardown leaves + # the fixture project and its global network and volumes behind on + # any runner that outlives the job. + down_all + fixture_compose down -v --remove-orphans >/dev/null 2>&1 || true exit 0 ;; -h|--help) echo "Usage: $0 [--skip-build] [--keep-running] [--only ] | down" @@ -125,12 +130,14 @@ log_info "==========================================" log_info "dstack-gateway cluster suite" log_info "==========================================" -rm -rf "$LOG_DIR" +# Discard anything a previous run left: node state is a bind mount that +# `compose down -v` does not remove and the host user cannot delete. +down_all +wipe_run_tree mkdir -p "$DATA_DIR" log_info "starting the attestation fixture" fixture_compose build >/dev/null -FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) fixture_compose up -d --wait >/dev/null "$SCRIPT_DIR/../build-gateway-image.sh" "$SCRIPT_DIR" $SKIP_BUILD From 0403ee2cc8f179c80d45643207be040321312e45 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:27:54 -0700 Subject: [PATCH 12/27] test(gateway): bound the cluster waits on wall clock, not iteration count `wait_for_instances` and `wait_for_digest_match` ran `timeout_seconds * 10` iterations of "probe, then sleep 0.1", on the assumption that an iteration costs 0.1s. Each probe forks a curl and a python3, so it costs about half again as much: a nominal 3s window measured 4.5s, and the two-probe digest loop is worse. That is not a rounding error. `test_push_fast_path` passes 3 to prove a push arrived before the 5s periodic sync could have done it anyway; the real window was ~4.8s against a 5s interval, so a dead push path was caught by the periodic round and the test still went green. Its own comment describes a bound the code did not implement. The slack cut both ways. `test_node_id_reuse_rejected` asked for 15s and passed only because it was really getting twice that: recovery there needs the rejected node's identity record to arrive in a sync *response* -- its own requests still fail the peer's inbound check -- and then an anti-entropy round to carry the store. Measured three times on an idle machine: 16s, 17s, 18s. With the timer made honest it passed one full run and failed the next, so the constant is raised to 40 with the measurement recorded next to it. `wait_for_debug` counted `sleep 1`s past a `docker compose port` and a curl, and is fixed the same way. --- dstack/gateway/test-run/cluster/lib.sh | 9 +++-- dstack/gateway/test-run/cluster/rpc.sh | 47 +++++++++++++++++------- dstack/gateway/test-run/cluster/tests.sh | 15 +++++++- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh index c0a4d0bd4..546a522e3 100644 --- a/dstack/gateway/test-run/cluster/lib.sh +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -263,9 +263,12 @@ stop_node() { wait_for_debug() { local node_id=$1 local timeout=${2:-60} - local waited=0 + # Wall clock, not an iteration count: each pass forks `docker compose port` + # and a curl, so counting `sleep 1`s overshot the stated timeout by half + # again. See `wait_until` in rpc.sh for the same fix on the sync waits. + local deadline=$((SECONDS + timeout)) local port - while [ "$waited" -lt "$timeout" ]; do + while [ "$SECONDS" -lt "$deadline" ]; do # Resolved inside the loop on purpose: docker only reports the mapping # once the container is running, so looking it up once up front would # leave every later request aimed at an empty port. @@ -274,7 +277,7 @@ wait_for_debug() { -H 'Content-Type: application/json' -d '{}' >/dev/null 2>&1; then return 0 fi - sleep 1; waited=$((waited + 1)) + sleep 1 done return 1 } diff --git a/dstack/gateway/test-run/cluster/rpc.sh b/dstack/gateway/test-run/cluster/rpc.sh index 0c125f08e..eccd77650 100644 --- a/dstack/gateway/test-run/cluster/rpc.sh +++ b/dstack/gateway/test-run/cluster/rpc.sh @@ -156,16 +156,42 @@ setup_peers() { done } +# Wall-clock deadline, not an iteration count. +# +# These loops used to run `$((timeout_seconds * 10))` iterations of "probe, then +# sleep 0.1", on the assumption that an iteration costs 0.1s. Every probe forks +# a curl and a python3, so an iteration costs closer to 0.15s and the real +# window was ~1.5x what the caller asked for. That is not a rounding error here: +# `test_push_fast_path` passes 3 to prove a push arrived *before* the 5s +# periodic sync could have done it anyway, and the true window was ~4.8s -- so a +# completely dead push path was caught by the periodic round and the test still +# went green. Bound on time and the number means what it says. +wait_until() { + local timeout_seconds=$1; shift + local deadline=$((SECONDS + timeout_seconds)) + while :; do + "$@" && return 0 + [ "$SECONDS" -lt "$deadline" ] || return 1 + sleep 0.1 + done +} + +_has_n_instances() { + [ "$(get_n_instances "$1")" -ge "$2" ] 2>/dev/null +} + wait_for_instances() { local node_id=$1 local expected=$2 local timeout_seconds=$3 - local _ - for _ in $(seq 1 $((timeout_seconds * 10))); do - [ "$(get_n_instances "$node_id")" -ge "$expected" ] 2>/dev/null && return 0 - sleep 0.1 - done - return 1 + wait_until "$timeout_seconds" _has_n_instances "$node_id" "$expected" +} + +_digests_match() { + local d1 d2 + d1=$(get_store_digest "$2" "$1") + d2=$(get_store_digest "$3" "$1") + [ -n "$d1" ] && [ "$d1" = "$d2" ] } wait_for_digest_match() { @@ -173,14 +199,7 @@ wait_for_digest_match() { local node_a=$2 local node_b=$3 local timeout_seconds=$4 - local d1 d2 _ - for _ in $(seq 1 $((timeout_seconds * 10))); do - d1=$(get_store_digest "$node_a" "$store") - d2=$(get_store_digest "$node_b" "$store") - if [ -n "$d1" ] && [ "$d1" = "$d2" ]; then return 0; fi - sleep 0.1 - done - return 1 + wait_until "$timeout_seconds" _digests_match "$store" "$node_a" "$node_b" } get_n_keys() { diff --git a/dstack/gateway/test-run/cluster/tests.sh b/dstack/gateway/test-run/cluster/tests.sh index a6f590df5..98d96d1f0 100644 --- a/dstack/gateway/test-run/cluster/tests.sh +++ b/dstack/gateway/test-run/cluster/tests.sh @@ -528,7 +528,20 @@ test_node_id_reuse_rejected() { wait_for_instances 2 1 20 || { log_error "fresh node did not recover after the UUID rejection"; return 1; } - wait_for_digest_match persistent 1 2 15 || { + # 40s, not the 15 this asked for before. + # + # Recovery here needs the rejected node's fresh identity record to reach + # node 1 in a sync *response* -- its own requests still fail node 1's + # inbound check -- and then an anti-entropy round to carry the store, so it + # costs several 5s intervals rather than one. Measured three times on an + # idle machine: 16s, 17s, 18s. + # + # It passed at 15 only because the wait helpers used to count iterations + # instead of seconds and so ran roughly twice as long as they claimed (see + # `wait_until` in rpc.sh). With the timer made honest this became a coin + # flip -- it passed on one full run and failed on the next. The number now + # says what the operation needs, with headroom for a loaded runner. + wait_for_digest_match persistent 1 2 40 || { log_error "stores did not converge after UUID recovery"; return 1; } verify_register_response \ From a937d8b8e7c6f7f3645446142c60d85db109bd3b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:27:54 -0700 Subject: [PATCH 13/27] test(gateway): fail closed where the cluster suite failed open Four places where a failure of the harness read as a pass of the test. `no_peer_was_rejected` is `! grep -q` over a file `dump_log` produces with `|| true`, and `! grep -q` on an empty file is true -- so it passed whenever the log could not be collected at all, which is the failure a negative assertion is most exposed to. It now requires the log to be non-empty first. `wipe_data_keeping_uuid` short-circuits its `rm` if `node_uuid` is not there, and errexit is off inside a test body -- they run as the condition of an `if` -- so the unchecked call let `test_bootstrap_after_data_dir_loss` "recover" a store that was never lost. Both wipes now fail loudly and both call sites check them. `--only` with a name that is not in ALL_TESTS selected nothing, left TESTS_FAILED at 0 and exited 0: a green run of zero tests. --- dstack/gateway/test-run/cluster/lib.sh | 19 ++++++++++++++----- .../test-run/cluster/run-cluster-tests.sh | 15 +++++++++++++++ dstack/gateway/test-run/cluster/tests.sh | 4 ++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dstack/gateway/test-run/cluster/lib.sh b/dstack/gateway/test-run/cluster/lib.sh index 546a522e3..5a457c527 100644 --- a/dstack/gateway/test-run/cluster/lib.sh +++ b/dstack/gateway/test-run/cluster/lib.sh @@ -319,11 +319,17 @@ data_op() { # Wipe a stopped node's store but let it keep the identity it already published, # which is what a node looks like after losing its disk but not its config. +# Fails loudly. The `&&` after the first `cp` short-circuits the wipe if +# node_uuid is not there, and errexit is off inside a test body (they run as the +# condition of an `if`), so an unchecked call let +# `test_bootstrap_after_data_dir_loss` "recover" a store that was never lost. wipe_data_keeping_uuid() { local node_id=$1 - data_op "cp /data/node${node_id}/wavekv/node_uuid /tmp/uuid && - rm -rf /data/node${node_id}/wavekv/* /data/node${node_id}/wavekv/.[!.]* 2>/dev/null; - cp /tmp/uuid /data/node${node_id}/wavekv/node_uuid" + data_op "set -e + cp /data/node${node_id}/wavekv/node_uuid /tmp/uuid + rm -rf /data/node${node_id}/wavekv/* /data/node${node_id}/wavekv/.[!.]* 2>/dev/null || true + cp /tmp/uuid /data/node${node_id}/wavekv/node_uuid" \ + || { log_error "could not wipe node ${node_id}'s store; the test below would prove nothing"; return 1; } } # Wipe a stopped node completely, identity included -- a stranger turning up @@ -331,6 +337,9 @@ wipe_data_keeping_uuid() { # bind-mount source, and replacing it swaps the inode the mount was set against. wipe_data() { local node_id=$1 - data_op "rm -rf /data/node${node_id}/..?* /data/node${node_id}/.[!.]* /data/node${node_id}/*" \ - 2>/dev/null || true + # Same reasoning as `wipe_data_keeping_uuid`: a silently skipped wipe leaves + # the following assertions testing the wrong starting state. The globs are + # allowed to match nothing; the operation as a whole is not allowed to fail. + data_op "rm -rf /data/node${node_id}/..?* /data/node${node_id}/.[!.]* /data/node${node_id}/* 2>/dev/null; exit 0" \ + || { log_error "could not wipe node ${node_id}'s data directory"; return 1; } } diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh index 1cfd70224..36784cd67 100755 --- a/dstack/gateway/test-run/cluster/run-cluster-tests.sh +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -121,6 +121,11 @@ cert_carries_app_id() { no_peer_was_rejected() { local node_id=$1 log log=$(dump_log "$node_id") + # `dump_log` ends in `|| true`, and `! grep -q` on an empty file is true, so + # without this the assertion passed whenever the log could not be collected + # at all -- the failure mode a negative assertion is most exposed to. Prove + # there is something to have searched first. + [ -s "$log" ] || { log_error "node ${node_id} produced no log to check"; return 1; } ! grep -qE "does not contain app_id|invalid quote|app_id mismatch" "$log" } @@ -222,6 +227,16 @@ ALL_TESTS=(test_persistence test_status_endpoint test_prpc_register \ test_admin_set_node_url test_admin_set_node_status \ test_node_status_register_exclude test_node_status_register_reject) +# An unknown --only name used to select nothing, leave TESTS_FAILED at 0 and +# exit 0: a green run of zero tests, which is the one result this suite must +# never produce. +if [ -n "$ONLY" ]; then + printf '%s\n' "${ALL_TESTS[@]}" | grep -qx -- "$ONLY" || { + log_error "unknown test: $ONLY" + exit 1 + } +fi + for t in "${ALL_TESTS[@]}"; do [ -n "$ONLY" ] && [ "$t" != "$ONLY" ] && continue run_ported "$t" diff --git a/dstack/gateway/test-run/cluster/tests.sh b/dstack/gateway/test-run/cluster/tests.sh index 98d96d1f0..cb4912a55 100644 --- a/dstack/gateway/test-run/cluster/tests.sh +++ b/dstack/gateway/test-run/cluster/tests.sh @@ -263,7 +263,7 @@ test_bootstrap_after_data_dir_loss() { log_error "node 2 did not report its identity before recovery"; return 1; } stop_node 2 - wipe_data_keeping_uuid 2 + wipe_data_keeping_uuid 2 || return 1 start_node 2 || return 1 wait_for_instances 2 1 15 || { @@ -490,7 +490,7 @@ test_node_id_reuse_rejected() { stop_node 2 - wipe_data 2 + wipe_data 2 || return 1 start_node 2 || return 1 new_uuid=$(get_node_uuid 2) From 4f20138cfc7de0ab1c0304adc65791d905cd1f32 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:27:54 -0700 Subject: [PATCH 14/27] test(gateway): probe the shared fixture before arming teardown All three suites decide whether to tear the shared attestation fixture down by whether they were the ones who started it. All three read that *after* installing the EXIT trap and next to the `up`, with a multi-minute image build in between -- so any failure in that window ran cleanup with the variable unset, took the `:-0` default, concluded it had started the fixture, and removed one another suite was using. That is the "network dstack-attestation declared as external, but could not be found" the comment there describes. Reading it before the trap is armed closes the window. The `down` subcommands also take the fixture now: it is a project of its own, so a suite-only teardown left it and its globally-named network and volumes on any runner that outlives the job. --- .../test-run/cluster/run-cluster-tests.sh | 10 ++++++++++ dstack/gateway/test-run/e2e/run-e2e.sh | 19 ++++++++++++------- .../test-run/proxy-e2e/run-proxy-tests.sh | 5 +++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/dstack/gateway/test-run/cluster/run-cluster-tests.sh b/dstack/gateway/test-run/cluster/run-cluster-tests.sh index 36784cd67..61d79ee3e 100755 --- a/dstack/gateway/test-run/cluster/run-cluster-tests.sh +++ b/dstack/gateway/test-run/cluster/run-cluster-tests.sh @@ -63,6 +63,16 @@ fi # directory lock above is the correct primitive -- it is race-free and needs no # pattern matching. +# Probed BEFORE the trap is installed, not next to the `up` that follows it. +# +# The fixture is shared with the other two suites and is meant to outlive any +# one of them, so cleanup tears it down only if this run started it. Reading +# that after `trap cleanup EXIT` meant every failure in between -- an image +# build, a wipe, a `docker` that is not there -- ran cleanup with the variable +# unset, took the `:-0` default, concluded it had started the fixture, and +# removed a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + cleanup() { $KEEP_RUNNING && return 0 # The last test's project is still up; the fixture outlives them all. diff --git a/dstack/gateway/test-run/e2e/run-e2e.sh b/dstack/gateway/test-run/e2e/run-e2e.sh index e60e6af34..c7c994540 100755 --- a/dstack/gateway/test-run/e2e/run-e2e.sh +++ b/dstack/gateway/test-run/e2e/run-e2e.sh @@ -100,17 +100,23 @@ if ! flock -n 9; then exit 1 fi +# Probed BEFORE the trap is installed, not next to the `up` that follows it: any +# failure in between would otherwise run cleanup with the variable unset, take +# the `:-0` default, and tear down a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + +# shellcheck disable=SC2317 # the body runs from the EXIT trap installed below cleanup() { if ! $KEEP_RUNNING; then log_info "Stopping containers..." docker compose down -v --remove-orphans 2>/dev/null || true # Only if this run started it. The fixture is meant to outlive a single - # suite; tearing down one this run found already up is what made the next - # run fail with "network dstack-attestation declared as external, but could - # not be found". - if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then - fixture_compose down -v --remove-orphans 2>/dev/null || true - fi + # suite; tearing down one this run found already up is what made the + # next run fail with "network dstack-attestation declared as external, + # but could not be found". + if [ "${FIXTURE_WAS_UP:-0}" -eq 0 ]; then + fixture_compose down -v --remove-orphans 2>/dev/null || true + fi fi } @@ -187,7 +193,6 @@ docker compose build fixture_compose build log_info "Starting the attestation fixture..." -FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) fixture_compose up -d --wait docker compose up -d mock-cf-dns-api pebble diff --git a/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh index 228fd78d0..292624aa5 100755 --- a/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh +++ b/dstack/gateway/test-run/proxy-e2e/run-proxy-tests.sh @@ -67,6 +67,11 @@ mkdir -p "$SCRIPT_DIR/run" docker run --rm -v "$SCRIPT_DIR/run:/r" alpine:latest \ find /r -mindepth 1 -delete >/dev/null 2>&1 || true +# Probed BEFORE the trap is installed. Reading it next to the `up` further down +# meant any failure in between ran cleanup with the variable unset, took the +# `:-0` default, and tore down a fixture another suite was using. +FIXTURE_WAS_UP=$(fixture_compose ps -q 2>/dev/null | wc -l) + cleanup() { compose down -v --remove-orphans >/dev/null 2>&1 || true # Only if this run started it. The fixture is meant to outlive a single From 25ed336b4590971df643ff3458612ba9b2a8870d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 26 Aug 2026 21:27:54 -0700 Subject: [PATCH 15/27] test(gateway): run the no-TLS-ULP arm even when the main one fails `set -e` is on, so `compose run --rm proxy-tests` followed by `MAIN_RC=$?` could only ever record 0 -- the script had already exited -- and the final `[ $MAIN_RC -eq 0 ] && [ $NOTLS_RC -eq 0 ]` was a tautology. The exit status still propagated, but the second arm was skipped on exactly the runs where something was already wrong, and that arm is the only place the kTLS truncation regression is covered. run-e2e.sh in this same series has the correct form. Two things that made a failure undiagnosable go with it: Both arms ran the whole suite against one `WORK=/work`, so the second overwrote every `gw-