From 71697ebca91666b45c1d4a80c2958cd04bec984e Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 18:12:09 -0700 Subject: [PATCH 01/10] fix(connect): remove tunnels after hosts go offline --- .../src/cloud/ManagedEndpointRuntime.ts | 7 + apps/server/src/cloud/http.test.ts | 126 +- apps/server/src/cloud/http.ts | 103 +- apps/server/src/server.test.ts | 15 + apps/server/src/server.ts | 60 +- docs/internals/t3-connect.md | 14 + docs/user/remote-access.md | 4 + .../migration.sql | 1 + .../snapshot.json | 1516 +++++++++++++++++ infra/relay/src/deploymentConfig.test.ts | 4 + infra/relay/src/deploymentConfig.ts | 6 +- .../environments/EnvironmentConnector.test.ts | 2 + .../ManagedEndpointAllocations.test.ts | 85 + .../ManagedEndpointAllocations.ts | 71 +- .../ManagedEndpointProvider.test.ts | 35 + .../environments/ManagedEndpointProvider.ts | 38 +- .../ManagedEndpointReaper.test.ts | 419 +++++ .../src/environments/ManagedEndpointReaper.ts | 166 ++ infra/relay/src/http/Api.test.ts | 162 +- infra/relay/src/http/Api.ts | 76 +- infra/relay/src/persistence/schema.ts | 1 + infra/relay/src/worker.ts | 49 +- packages/contracts/src/relay.ts | 20 + 23 files changed, 2924 insertions(+), 56 deletions(-) create mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql create mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json create mode 100644 infra/relay/src/environments/ManagedEndpointReaper.test.ts create mode 100644 infra/relay/src/environments/ManagedEndpointReaper.ts diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 89c0a23783c0..1bbdd5f08e69 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as PubSub from "effect/PubSub"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -58,6 +59,8 @@ export class CloudManagedEndpointRuntime extends Context.Service< readonly applyConfig: ( config: RelayManagedEndpointRuntimeConfig | null, ) => Effect.Effect; + readonly recoveryRequests: Stream.Stream; + readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -104,6 +107,7 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); + const recoveryRequests = yield* PubSub.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; @@ -144,6 +148,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); + yield* PubSub.publish(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -305,6 +310,8 @@ export const make = Effect.gen(function* () { const runtime = CloudManagedEndpointRuntime.of({ applyConfig, + recoveryRequests: Stream.fromPubSub(recoveryRequests), + requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), }); const initialConfig = yield* readRuntimeConfig.pipe( diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 0f24e6f34176..2f217f7bed0a 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -7,6 +7,7 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Tracer from "effect/Tracer"; +import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, @@ -30,13 +31,20 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts"; +import { + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeRuntimeConfig, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_URL_SECRET, +} from "./config.ts"; import { consumeCloudReplayGuards, isSupportedLinkProviderKind, linkProofScopes, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -209,6 +217,8 @@ describe("reconcileDesiredCloudLink", () => { ManagedEndpointRuntime.CloudManagedEndpointRuntime, ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: unusedSecretStoreOperation, + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -303,10 +313,18 @@ describe("releaseManagedTunnelOnShutdown", () => { applyConfig: (config) => Effect.sync(() => { harness.applyConfigCalls.push(config); - return { - status: "disabled", - } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus; + return config === null + ? ({ + status: "disabled", + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus) + : ({ + status: "running", + providerKind: "cloudflare_tunnel", + pid: 123, + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus); }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, }), ), Effect.provideService( @@ -579,6 +597,106 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ); }); + + it.effect("recovers a web-linked tunnel with its environment credential", () => { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel", + connectorToken: "new-token", + tunnelId: "new-tunnel", + } as const; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel"); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + expect(applyConfigCalls).toEqual([nextConfig]); + expect( + Option.getOrNull( + decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))), + ), + ).toEqual(nextConfig); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + }), + ); + }); + + it.effect("does not recover an environment without a managed tunnel credential", () => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("keeps a tunnel configuration replaced during recovery", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const freshConfig = new TextEncoder().encode("fresh-config"); + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); + return Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + }, + }); + }, + }), + ); + }); }); describe("link proof provider kinds", () => { diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 29fdfe8ece2f..4d40c5d59299 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,6 +28,7 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryResponse, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -453,6 +454,7 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, + options?: { readonly requestRecovery?: boolean }, ) { yield* validateRelayConfigPayload(payload); yield* validateLinkedCloudUser({ @@ -489,6 +491,9 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); + } } else { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); } @@ -607,14 +612,18 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi schema: RelayEnvironmentLinkResponse, }); yield* setCliDesiredCloudLink(true, mode); - return yield* applyCloudRelayConfig(dependencies, { - relayUrl, - relayIssuer: link.relayIssuer, - cloudUserId: link.cloudUserId, - environmentCredential: link.environmentCredential, - cloudMintPublicKey: link.cloudMintPublicKey, - endpointRuntime: link.endpointRuntime, - }); + return yield* applyCloudRelayConfig( + dependencies, + { + relayUrl, + relayIssuer: link.relayIssuer, + cloudUserId: link.cloudUserId, + environmentCredential: link.environmentCredential, + cloudMintPublicKey: link.cloudMintPublicKey, + endpointRuntime: link.endpointRuntime, + }, + { requestRecovery: false }, + ); }, Effect.catchIf( ServerSecretStore.isSecretStoreError, @@ -635,6 +644,84 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( + function* (localOrigin: string) { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + + const localUrl = yield* Effect.try({ + try: () => new URL(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + if (localUrl.origin !== localOrigin) { + return yield* new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }); + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const recovered = yield* relayClientRequest(dependencies, { + url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: bytesToString(cloudUserId.value), + origin: { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }, + }, + schema: RelayManagedEndpointRecoveryResponse, + }); + if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { + return yield* new EnvironmentHttpInternalServerError({ + message: "T3 Connect returned an unsupported managed tunnel configuration.", + }); + } + + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } + + const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + return true; + }, +); + // The launcher owns this durable state, so read it directly both when a trial // decides whether it owns pre-activation cleanup and while a server tears down. export const pendingServiceUpdateExists = Effect.gen(function* () { diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5e4f19172eff..08a639cd20c5 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -943,6 +943,8 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, ...options?.layers?.cloudManagedEndpointRuntime, }), ), @@ -2509,6 +2511,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("unlinks local cloud state and disables the managed endpoint runtime", () => Effect.gen(function* () { const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { @@ -2525,6 +2528,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => { ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), }); }, + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), }, }, }); @@ -2599,6 +2606,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); + assert.deepEqual(requestedRecoveryConfigs, [ + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-id", + tunnelName: "tunnel-name", + }, + ]); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0a31bf376dae..8762d405cdc6 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -4,6 +4,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -92,6 +93,7 @@ import { connectHttpApiLayer, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; @@ -613,22 +615,13 @@ export const makeServerLayer = Layer.unwrap( if (!cleanupBeforeActivation) { yield* Effect.addFinalizer(() => releaseManagedTunnel); } - if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; - // No settling delay before the first attempt: routes are already - // serving by the time activation opens this gate (the startup - // sequence awaits routesReady), and the retry schedule below - // covers anything this sleep used to hedge against. Every - // millisecond here is dead time on the path to remote - // reachability after a restart. - yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( + const localOrigin = `http://127.0.0.1:${address.port}`; + const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const recoverManagedTunnel = recoverManagedCloudTunnel(localOrigin).pipe( Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), @@ -636,13 +629,46 @@ export const makeServerLayer = Layer.unwrap( Schedule.upTo({ duration: "10 minutes" }), ), }), - Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), - Effect.catch((cause) => - Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, - }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ); + yield* endpointRuntime.recoveryRequests.pipe( + Stream.runForEach(() => recoverManagedTunnel), + Effect.forkScoped, + ); + // No settling delay before the first attempt: routes are already + // serving by the time activation opens this gate (the startup + // sequence awaits routesReady), and the retry schedule below + // covers anything this sleep used to hedge against. Every + // millisecond here is dead time on the path to remote + // reachability after a restart. + if (yield* CloudCliState.readCliDesiredCloudLink) { + yield* reconcileDesiredCloudLink(localOrigin).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { + cause, + }), + ), + ); + } + yield* recoverManagedTunnel; }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 6f796123e98b..e342f5adba62 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -125,6 +125,20 @@ connector, and attempts to revoke the relay-side environment record. It retains authorization so `t3 connect link` can re-enable exposure without another browser flow. `t3 connect logout` performs the same cleanup and removes the stored CLI authorization. +### Managed tunnel lifecycle + +Every linked environment stores a relay-issued environment credential. When a managed environment +starts or its connector exits, the server uses that credential to request a tunnel from the relay. +This also covers environments linked through web or mobile settings, which do not have a stored CLI +credential. The relay keeps the existing hostname and DNS record, so a replacement tunnel does not +change the public endpoint. + +After a host completes this recovery request, the relay records that the host can recreate its own +tunnel. The existing five-minute maintenance job removes tunnels from those hosts when Cloudflare +reports that they have been down for at least five minutes. Tunnels that never connected are removed +when they are at least five minutes old. The job leaves older hosts alone until they complete a +recovery request, and it only removes tunnels that belong to its own deployment stage. + The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 5993fca5b352..ff6fc4285fb5 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -229,6 +229,10 @@ works for a server that was wiped or is no longer reachable. Device-local connec controls remain in **Settings** → **Connections** on web and desktop or **Settings** → **Environments** on mobile. +If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. +The environment stays linked to your account and keeps the same address. When the server starts +again, T3 Connect creates a replacement tunnel automatically. You do not need to pair it again. + ## Security Notes - Treat pairing URLs and pairing tokens like passwords. diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql new file mode 100644 index 000000000000..f528c09fbc7f --- /dev/null +++ b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json new file mode 100644 index 000000000000..648638a0e17d --- /dev/null +++ b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json @@ -0,0 +1,1516 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "f3c6f2a5-2ecf-43cb-b381-98790fdca66e", + "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/deploymentConfig.test.ts b/infra/relay/src/deploymentConfig.test.ts index 44c7627a4daf..175e36ca527d 100644 --- a/infra/relay/src/deploymentConfig.test.ts +++ b/infra/relay/src/deploymentConfig.test.ts @@ -7,6 +7,7 @@ import { managedEndpointHostname, isManagedEndpointHostname, managedEndpointTunnelName, + managedEndpointTunnelNamePrefix, relayOwnsManagedEndpointZone, RelayPublicDomainLabelTooLongError, relayPublicDomainForStage, @@ -91,6 +92,9 @@ describe("managed endpoint names", () => { expect(managedEndpointTunnelName("dev_julius", hash)).toBe( "t3coderelay-managedendpoint-dev-julius-abcdef0123456789", ); + expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe( + "t3coderelay-managedendpoint-dev-julius-", + ); }); it("keeps the DNS label within the provider limit for long stage names", () => { diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index fe9d37b29988..1dab362a9f09 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi }; } +export function managedEndpointTunnelNamePrefix(stage: string): string { + return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`; +} + export function managedEndpointTunnelName(stage: string, hash: string): string { - return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`; + return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 7f536bafb375..dfa809a9f151 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -197,6 +197,8 @@ function makeAllocations( recordTunnel: () => Effect.die("unused"), recordDns: () => Effect.die("unused"), markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ebf51de100c1..c4c06e9e723e 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -10,6 +10,91 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("records recovery support and advances the allocation generation", () => { + let updated: + | { + readonly recoveryEnabledAt: string; + readonly updatedAt: string; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { + updated = values; + return { + where: () => Effect.void, + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + }); + + expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); + expect(updated?.recoveryEnabledAt).toBeDefined(); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("returns recovery support with tunnel allocation lookups", () => { + const base = { + userId: "user-1", + hostname: "environment.example.test", + tunnelName: "managed-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-08-25T12:00:00.000Z", + updatedAt: "2026-08-25T12:00:00.000Z", + }; + const fakeDb = { + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + }, + ]), + }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.listByTunnelNames(["first-tunnel", "second-tunnel"]); + + expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ + ["tunnel-1", true], + ["tunnel-2", false], + ]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("skips the database for an empty tunnel lookup", () => + Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect(yield* allocations.listByTunnelNames([])).toEqual([]); + }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), + ); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { let claimedAt: string | undefined; const fakeDb = { diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4320eeea3b72..4fbdadd20239 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,5 +1,5 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray, isNull } from "drizzle-orm"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -25,6 +25,10 @@ export interface ManagedEndpointAllocation { readonly updatedAt: string; } +export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { + readonly recoveryEnabled: boolean; +} + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -50,6 +54,8 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "record-tunnel", "record-dns", "mark-ready", + "enable-recovery", + "list-tunnels", "claim-release", "claim-deprovision", "remove", @@ -119,6 +125,15 @@ export class ManagedEndpointAllocations extends Context.Service< readonly markReady: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; + readonly enableRecovery: ( + input: ManagedEndpointAllocationKey, + ) => Effect.Effect; + readonly listByTunnelNames: ( + tunnelNames: ReadonlyArray, + ) => Effect.Effect< + ReadonlyArray, + ManagedEndpointAllocationPersistenceError + >; /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the @@ -312,6 +327,60 @@ export const make = Effect.gen(function* () { ), ); }), + enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( + input: ManagedEndpointAllocationKey, + ) { + const now = DateTime.formatIso(yield* DateTime.now); + yield* db + .update(relayManagedEndpointAllocations) + .set({ recoveryEnabledAt: now, updatedAt: now }) + .where( + and(whereAllocation(input), isNull(relayManagedEndpointAllocations.recoveryEnabledAt)), + ) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "enable-recovery", + stage: "database-request", + ...input, + cause, + }), + ), + ); + }), + listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")( + function* (tunnelNames: ReadonlyArray) { + if (tunnelNames.length === 0) { + return []; + } + return yield* db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + }) + .from(relayManagedEndpointAllocations) + .where(inArray(relayManagedEndpointAllocations.tunnelName, tunnelNames)) + .pipe( + Effect.map((rows) => + rows.map(({ recoveryEnabledAt, ...allocation }) => ({ + ...allocation, + recoveryEnabled: recoveryEnabledAt !== null, + })), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ); + }, + ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 4d136658c8fd..8ec3f3762b75 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -159,6 +159,7 @@ function makeDnsClient( function makeAllocations(calls: AllocationCall[] = []) { const allocations = new Map(); + const recoveryEnabled = new Set(); let generation = 0; const mutate = ( key: string, @@ -214,6 +215,19 @@ function makeAllocations(calls: AllocationCall[] = []) { readyAt: "2026-06-02T00:00:00.000Z", })); }), + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabled.add(allocationKey(input)); + }), + listByTunnelNames: (tunnelNames) => + Effect.sync(() => + [...allocations.values()] + .filter((allocation) => tunnelNames.includes(allocation.tunnelName)) + .map((allocation) => ({ + ...allocation, + recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + })), + ), claimRelease: (input) => Effect.sync(() => { calls.push({ operation: "claimRelease", input }); @@ -939,6 +953,27 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not release a tunnel when the requested tunnel id is outdated", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + makeAllocations(), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 9a578874844e..1789c014ae5d 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -166,13 +166,28 @@ export class ManagedEndpointProvider extends Context.Service< readonly release: (input: { readonly userId: string; readonly environmentId: string; + readonly expectedTunnelId?: string; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} -interface ManagedEndpointTunnel { +export interface ManagedEndpointTunnel { readonly id?: string | null; readonly name?: string | null; + readonly status?: string | null; + readonly createdAt?: string | null; + readonly connsInactiveAt?: string | null; +} + +export interface ManagedEndpointTunnelListRequest { + readonly isDeleted: false; + readonly name?: string; + readonly includePrefix?: string; + readonly status?: "inactive" | "down"; + readonly existedAt?: string; + readonly wasInactiveAt?: string; + readonly page?: number; + readonly perPage?: number; } const ManagedEndpointTunnelClientOperation = Schema.Literals([ @@ -201,11 +216,15 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass Effect.Effect< - { readonly result: ReadonlyArray }, + readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< + { + readonly result: ReadonlyArray; + readonly resultInfo?: { + readonly page?: number | null; + readonly perPage?: number | null; + readonly totalCount?: number | null; + } | null; + }, ManagedEndpointTunnelClientError >; readonly create: (request: { @@ -554,6 +573,9 @@ export const make = Effect.gen(function* () { if (allocation === null || tunnelId === null) { return true; } + if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { + return false; + } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment // restart) rewrites updatedAt when it records its tunnel, so a stale @@ -865,7 +887,7 @@ export const layerCloudflareBindings = ( alchemyRuntimeContext: Alchemy.BaseRuntimeContext, ) => layer.pipe( - Layer.provide( + Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ list: (request) => @@ -874,7 +896,7 @@ export const layerCloudflareBindings = ( (cause) => new ManagedEndpointTunnelClientError({ operation: "list", - tunnelName: request.name, + ...(request.name === undefined ? {} : { tunnelName: request.name }), cause, }), ), diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts new file mode 100644 index 000000000000..c8e4816b348e --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as TestClock from "effect/testing/TestClock"; + +import * as RelayConfiguration from "../Config.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts"; + +const NOW = "2026-08-25T12:00:00.000Z"; +const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds; +const PREFIX = "t3coderelay-managedendpoint-prod-"; + +function tunnel(input: { + readonly id: string; + readonly suffix: string; + readonly status: "down" | "inactive" | "healthy" | "degraded"; + readonly timestamp?: string | null; + readonly prefix?: string; +}): ManagedEndpointProvider.ManagedEndpointTunnel { + return { + id: input.id, + name: `${input.prefix ?? PREFIX}${input.suffix}`, + status: input.status, + ...(input.timestamp === undefined + ? {} + : input.status === "inactive" + ? { createdAt: input.timestamp } + : { connsInactiveAt: input.timestamp }), + }; +} + +function allocation(input: { + readonly tunnelId: string; + readonly recoveryEnabled: boolean; +}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { + return { + userId: "user-1", + environmentId: `environment-${input.tunnelId}`, + hostname: `${input.tunnelId}.example.test`, + tunnelId: input.tunnelId, + tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, + dnsRecordId: "dns-1", + readyAt: "2026-08-25T11:00:00.000Z", + updatedAt: "2026-08-25T11:00:00.000Z", + recoveryEnabled: input.recoveryEnabled, + }; +} + +function harness(input?: { + readonly tunnels?: ReadonlyArray; + readonly allocations?: ReadonlyArray; + readonly namespace?: string; + readonly failTunnelId?: string; + readonly skipTunnelId?: string; +}) { + const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; + const deleted: string[] = []; + const releases: Array<{ + readonly userId: string; + readonly environmentId: string; + readonly expectedTunnelId?: string; + }> = []; + const recorded = (input?.allocations ?? []).map((entry) => { + const matching = input?.tunnels?.find((candidate) => candidate.id === entry.tunnelId); + return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; + }); + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + list: (request) => + Effect.sync(() => { + listRequests.push(request); + const matching = (input?.tunnels ?? []).filter((entry) => entry.status === request.status); + const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); + return { + result: matching.slice(start, start + (request.perPage ?? 100)), + resultInfo: { + page: request.page ?? 1, + perPage: request.perPage ?? 100, + totalCount: matching.length, + }, + }; + }), + create: () => Effect.die("unused"), + putConfiguration: () => Effect.die("unused"), + getToken: () => Effect.die("unused"), + delete: (tunnelId) => + tunnelId === input?.failTunnelId + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause: "Cloudflare refused the deletion", + }), + ) + : Effect.sync(() => { + deleted.push(tunnelId); + }), + }); + const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + get: () => Effect.die("unused"), + reserve: () => Effect.die("unused"), + recordTunnel: () => Effect.die("unused"), + recordDns: () => Effect.die("unused"), + markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: (tunnelNames) => + Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), + claimRelease: () => Effect.die("unused"), + claimDeprovision: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + removeClaimed: () => Effect.die("unused"), + }); + const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({ + provision: () => Effect.die("unused"), + prepareDeprovision: () => Effect.die("unused"), + deprovision: () => Effect.die("unused"), + release: (request) => + Effect.sync(() => { + releases.push(request); + if (request.expectedTunnelId === input?.skipTunnelId) { + return false; + } + if (request.expectedTunnelId !== undefined) { + deleted.push(request.expectedTunnelId); + } + return true; + }), + }); + const config = RelayConfiguration.RelayConfiguration.of({ + relayIssuer: "https://relay.example.test", + apns: { + environment: "sandbox", + teamId: "team-id", + keyId: "key-id", + privateKey: Redacted.make("private-key"), + bundleId: "com.t3tools.t3code.dev", + }, + apnsDeliveryJobSigningSecret: Redacted.make("job-secret"), + clerkSecretKey: Redacted.make("clerk-secret"), + clerkPublishableKey: "pk_test_test", + clerkJwtAudience: "t3-code-relay", + cloudMintPrivateKey: Redacted.make("cloud-private-key"), + cloudMintPublicKey: "cloud-public-key", + managedEndpointBaseDomain: "example.test", + managedEndpointNamespace: input?.namespace ?? "prod", + }); + + return { + listRequests, + deleted, + releases, + layer: ManagedEndpointReaper.layer.pipe( + Layer.provide( + Layer.mergeAll( + RelayConfiguration.layer(config), + ManagedEndpointProvider.layerTunnelClient(tunnelClient), + Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider), + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService), + ), + ), + ), + }; +} + +describe("ManagedEndpointReaper", () => { + it.effect("removes expired down and inactive tunnels from recoverable environments", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "down-1", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:00.000Z", + }), + tunnel({ + id: "inactive-1", + suffix: "bbbbbbbbbbbbbbbb", + status: "inactive", + timestamp: "2026-08-25T11:54:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "down-1", recoveryEnabled: true }), + allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 2, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual(["down-1", "inactive-1"]); + expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ + "down-1", + "inactive-1", + ]); + expect(state.listRequests).toEqual([ + { + isDeleted: false, + includePrefix: PREFIX, + status: "down", + existedAt: "2026-08-25T11:55:00.000Z", + wasInactiveAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + { + isDeleted: false, + includePrefix: PREFIX, + status: "inactive", + existedAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + ]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "recent", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:01.000Z", + }), + tunnel({ + id: "other-stage", + prefix: `${PREFIX}julius-`, + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "missing-time", + suffix: "cccccccccccccccc", + status: "inactive", + timestamp: null, + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 0, + deleted: 0, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps tunnels owned by environments that cannot recover yet", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "legacy", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 0, + skippedLegacy: 1, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("removes expired tunnels that no longer have an allocation", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "orphan", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["orphan"]); + expect(state.releases).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not count a tunnel that was replaced before its release", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "replaced", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })], + skipTunnelId: "replaced", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues after an orphan tunnel deletion fails", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "failed", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "next", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + failTunnelId: "failed", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 1, + skippedLegacy: 0, + failed: 1, + }); + expect(state.deleted).toEqual(["next"]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { + const entries = Array.from({ length: 101 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries.map((entry, index) => + allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 101, + deleted: 1, + skippedLegacy: 100, + failed: 0, + }); + expect(state.deleted).toEqual(["tunnel-100"]); + expect( + state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page), + ).toEqual([1, 2]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("limits each cleanup run to 100 tunnel deletions", () => { + const state = harness({ + tunnels: Array.from({ length: 105 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(100); + expect(state.deleted).toHaveLength(100); + }).pipe(Effect.provide(state.layer)); + }); +}); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts new file mode 100644 index 000000000000..cfd933ebe650 --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -0,0 +1,166 @@ +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import * as RelayConfiguration from "../Config.ts"; +import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; + +export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; +export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; +export const MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT = 100; + +export interface ManagedEndpointSweepResult { + readonly scanned: number; + readonly deleted: number; + readonly skippedLegacy: number; + readonly failed: number; +} + +export class ManagedEndpointReaper extends Context.Service< + ManagedEndpointReaper, + { + readonly sweep: Effect.Effect< + ManagedEndpointSweepResult, + | ManagedEndpointProvider.ManagedEndpointTunnelClientError + | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError + >; + } +>()("t3code-relay/environments/ManagedEndpointReaper") {} + +function isExpiredManagedTunnel(input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; +}): input is typeof input & { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; +} { + const { tunnel, status, prefix, cutoff } = input; + if ( + typeof tunnel.id !== "string" || + typeof tunnel.name !== "string" || + tunnel.status !== status || + !tunnel.name.startsWith(prefix) || + !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length)) + ) { + return false; + } + + const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; + if (typeof inactiveAt !== "string") { + return false; + } + const timestamp = DateTime.make(inactiveAt); + return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; +} + +export const make = Effect.gen(function* () { + const config = yield* RelayConfiguration.RelayConfiguration; + const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + + const sweep = Effect.gen(function* () { + const namespace = config.managedEndpointNamespace; + if (!namespace) { + return { scanned: 0, deleted: 0, skippedLegacy: 0, failed: 0 }; + } + + const now = yield* DateTime.now; + const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); + const cutoffIso = DateTime.formatIso(cutoff); + const prefix = managedEndpointTunnelNamePrefix(namespace); + let scanned = 0; + let deleted = 0; + let skippedLegacy = 0; + let failed = 0; + + for (const status of ["down", "inactive"] as const) { + let page = 1; + while (deleted < MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + const response = yield* tunnels.list({ + isDeleted: false, + includePrefix: prefix, + status, + existedAt: cutoffIso, + ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}), + page, + perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, + }); + const expired = response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => tunnel); + scanned += expired.length; + + const recorded = yield* allocations.listByTunnelNames(expired.map((tunnel) => tunnel.name)); + const recordedByTunnelId = new Map( + recorded + .filter((allocation) => allocation.tunnelId !== null) + .map((allocation) => [allocation.tunnelId, allocation]), + ); + + for (const tunnel of expired) { + if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + break; + } + const allocation = recordedByTunnelId.get(tunnel.id); + if (allocation !== undefined && !allocation.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + + const result = + allocation === undefined + ? yield* tunnels.delete(tunnel.id).pipe(Effect.as(true), Effect.result) + : yield* provider + .release({ + userId: allocation.userId, + environmentId: allocation.environmentId, + expectedTunnelId: tunnel.id, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + const totalCount = response.resultInfo?.totalCount; + if ( + response.result.length === 0 || + (typeof totalCount === "number" + ? page * MANAGED_ENDPOINT_SWEEP_PAGE_SIZE >= totalCount + : response.result.length < MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) + ) { + break; + } + page += 1; + } + } + + return { scanned, deleted, skippedLegacy, failed }; + }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); + + return ManagedEndpointReaper.of({ sweep }); +}); + +export const layer = Layer.effect(ManagedEndpointReaper, make); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index daf756a2b7cc..12b39dd3e7e9 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -23,6 +23,7 @@ import { relayDocsRedirectRoute, relayEnvironmentAuthLayer, relayNotFoundRoute, + recoverEnvironmentTunnelRecord, revokeEnvironmentLinkRecord, traceRelayHttpRequestWith, unlinkEnvironmentRecord, @@ -33,6 +34,7 @@ import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; vi.mock("@clerk/backend", () => ({ @@ -168,6 +170,7 @@ function relayUnlinkTestLayer(input?: { readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"]; readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; + readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; }) { return Layer.mergeAll( Layer.succeed( @@ -199,7 +202,7 @@ function relayUnlinkTestLayer(input?: { Layer.succeed( ManagedEndpointProvider.ManagedEndpointProvider, ManagedEndpointProvider.ManagedEndpointProvider.of({ - provision: () => Effect.die("unused provision"), + provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.void), release: () => Effect.die("unused release"), @@ -220,6 +223,163 @@ const linkedEnvironmentRecord = { linkedAt: "2026-07-28T00:00:00.000Z", } as const; +describe("relay managed tunnel recovery", () => { + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { + let recoveryEnabledFor: { readonly userId: string; readonly environmentId: string } | null = + null; + const runtime = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }; + + return Effect.gen(function* () { + expect( + yield* recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ + endpoint: linkedEnvironmentRecord.endpoint, + endpointRuntime: runtime, + }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + }), + }), + ), + ), + ); + }); + + it.effect("rejects a credential from a different environment owner", () => { + let provisioned = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(provisioned).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.sync(() => { + provisioned = true; + return { + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ); + }); + + it.effect("does not recover a publish-only environment", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.succeed({ + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ), + ); + + it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: { + httpBaseUrl: "https://different.example.test/", + wsBaseUrl: "wss://different.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + }), + }), + ), + ), + ); + }); +}); + describe("relay environment unlink", () => { it.effect("revokes the link and its credentials in one database transaction", () => { const calls: Array = []; diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 50bcff665a9b..001e0de7e7d3 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -46,6 +46,7 @@ import { RelayEnvironmentLinkLimitExceededError, RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, + type RelayManagedEndpointOrigin, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; @@ -464,6 +465,51 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +export const recoverEnvironmentTunnelRecord = Effect.fn( + "relay.api.server.recoverEnvironmentTunnelRecord", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + const recovered = yield* managedEndpointProvider.provision({ + userId: input.userId, + environmentId: input.environmentId, + origin: input.origin, + }); + if ( + recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || + recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + yield* allocations.enableRecovery({ + userId: input.userId, + environmentId: input.environmentId, + }); + return { + endpoint: recovered.endpoint, + endpointRuntime: recovered.runtime, + }; +}); + export const mobileApi = HttpApiBuilder.group( RelayApi, "mobile", @@ -856,7 +902,7 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; - return handlers.handle( + const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( function* (args) { @@ -984,6 +1030,34 @@ export const serverApi = HttpApiBuilder.group( mapRelayCommonApiErrors("not_authorized"), ), ); + + return activityHandlers.handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 61b72f2df868..ddd276ccfd70 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -93,6 +93,7 @@ export const relayManagedEndpointAllocations = pgTable( tunnelName: text("tunnel_name").notNull(), dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), + recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 77dfd845c5bc..2f09fd3e5b9c 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -55,6 +55,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts"; import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts"; import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"; import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; @@ -195,7 +196,9 @@ export const ApiLive = Api.make( Layer.provideMerge(AgentActivityPublisher.layer), Layer.provideMerge(EnvironmentConnector.layer), Layer.provideMerge(EnvironmentLinker.layer), - Layer.provideMerge(EnvironmentPublishSignatures.layer), + Layer.provideMerge( + Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer), + ), Layer.provideMerge( ManagedEndpointProvider.layerCloudflareBindings( managedEndpointTunnelBinding, @@ -261,22 +264,38 @@ export const ApiLive = Api.make( ); yield* Cloudflare.Workers.cron("*/5 * * * *", () => - DpopProofs.DpopProofReplay.pipe( - Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), - // Terminal thread rows are kept briefly so finished agents show as - // Done/Failed in the Live Activity; sweep them once they age out. - Effect.andThen( - Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( - Effect.flatMap(([activityRows, now]) => - activityRows.pruneTerminal({ - updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), - }), + Effect.all( + [ + DpopProofs.DpopProofReplay.pipe( + Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), + // Keep completed thread rows long enough to show their final state. + Effect.andThen( + Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( + Effect.flatMap(([activityRows, now]) => + activityRows.pruneTerminal({ + updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), + }), + ), + ), + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), - ), - Effect.withSpan("relay.cron.prune_expired_state"), - Effect.provide(runtimeLayer), - ), + ManagedEndpointReaper.ManagedEndpointReaper.pipe( + Effect.flatMap((reaper) => reaper.sweep), + Effect.tap((result) => + result.scanned > 0 + ? Effect.logInfo("Finished managed tunnel cleanup", result) + : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + ), + ), + ], + { concurrency: 2, discard: true }, + ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)), ); const fetch = Layer.merge( diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 52f7d7d43550..76b0b0ef7438 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -159,6 +159,18 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({ }); export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type; +export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, +}); +export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; + +export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ + endpoint: RelayManagedEndpoint, + endpointRuntime: RelayManagedEndpointRuntimeConfig, +}); +export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type; + export const RelayLinkProofRequest = Schema.Struct({ challenge: Schema.String, relayIssuer: Schema.String, @@ -1053,6 +1065,14 @@ export const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") export const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRequest, + success: RelayManagedEndpointRecoveryResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"), HttpApiEndpoint.post( "publishAgentActivity", "/v1/environments/:environmentId/threads/:threadId/agent-activity", From 26c5820ba996781c8de03c384e267c185ce7c17c Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 20:15:43 -0700 Subject: [PATCH 02/10] fix(connect): protect tunnel recovery and cleanup from races --- .../src/cloud/ManagedEndpointRuntime.test.ts | 30 ++++ .../src/cloud/ManagedEndpointRuntime.ts | 3 + apps/server/src/cloud/http.test.ts | 2 + apps/server/src/cloud/http.ts | 169 ++++++++++-------- apps/server/src/server.test.ts | 1 + apps/server/src/server.ts | 31 ++-- .../environments/EnvironmentLinker.test.ts | 3 +- .../ManagedEndpointAllocations.test.ts | 52 +++++- .../ManagedEndpointAllocations.ts | 46 +++-- .../ManagedEndpointProvider.test.ts | 115 +++++++++++- .../environments/ManagedEndpointProvider.ts | 103 +++++++++-- .../ManagedEndpointReaper.test.ts | 169 +++++++++++++++++- .../src/environments/ManagedEndpointReaper.ts | 148 +++++++++------ infra/relay/src/http/Api.test.ts | 117 +++++++++++- infra/relay/src/http/Api.ts | 28 ++- 15 files changed, 830 insertions(+), 187 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index b45b5099252a..54a12373c732 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -80,6 +80,36 @@ function makeHandle(input: { } describe("CloudManagedEndpointRuntime", () => { + it.effect("serializes updates to persisted cloud link state", () => + Effect.gen(function* () { + const firstEntered = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const secondEntered = yield* Deferred.make(); + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + + const first = yield* runtime + .withLinkStateLock( + Deferred.succeed(firstEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + ), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(firstEntered); + + const second = yield* runtime + .withLinkStateLock(Deferred.succeed(secondEntered, undefined)) + .pipe(Effect.forkChild); + expect(yield* Deferred.isDone(secondEntered)).toBe(false); + + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Deferred.isDone(secondEntered)).toBe(true); + }), + ); + it("classifies Cloudflare connection and warning output", () => { expect( ManagedEndpointRuntime.classifyRelayClientOutput( diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 1bbdd5f08e69..cf566ac0a4c8 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -61,6 +61,7 @@ export class CloudManagedEndpointRuntime extends Context.Service< ) => Effect.Effect; readonly recoveryRequests: Stream.Stream; readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -109,6 +110,7 @@ export const make = Effect.gen(function* () { const desiredConfigRef = yield* Ref.make(null); const recoveryRequests = yield* PubSub.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); + const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; const stopActive = Effect.gen(function* () { @@ -312,6 +314,7 @@ export const make = Effect.gen(function* () { applyConfig, recoveryRequests: Stream.fromPubSub(recoveryRequests), requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), + withLinkStateLock: linkStateSemaphore.withPermits(1), }); const initialConfig = yield* readRuntimeConfig.pipe( diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 2f217f7bed0a..fd0328b2c414 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -219,6 +219,7 @@ describe("reconcileDesiredCloudLink", () => { applyConfig: unusedSecretStoreOperation, recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -325,6 +326,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, }), ), Effect.provideService( diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 4d40c5d59299..9ae34e6d0d5c 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -456,48 +456,55 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu payload: RelayEnvironmentConfigRequest, options?: { readonly requestRecovery?: boolean }, ) { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, - }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, - }); - } + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, + }); + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + const ok = + endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; + if (!ok) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, + }); + } - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); + yield* dependencies.secrets.set( + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), + ); + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), + ); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); + } + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey)); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); - yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), - ); - if (options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( @@ -694,31 +701,35 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan }); } - const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if ( - Option.isNone(currentConfig) || - bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) - ) { - return false; - } + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } - const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); - if (status.status !== "running") { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus: status, - }); - } - const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( - Effect.mapError( - () => - new EnvironmentHttpInternalServerError({ - message: "Could not persist the recovered managed tunnel configuration.", - }), - ), + const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + return true; + }), ); - yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); - return true; }, ); @@ -875,21 +886,25 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")( const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( function* (dependencies: CloudHttpDependencies) { yield* requireEnvironmentScope(AuthRelayWriteScope); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); - yield* Effect.all( - [ - dependencies.secrets.remove(CLOUD_LINKED_USER_ID), - dependencies.secrets.remove(RELAY_URL_SECRET), - dependencies.secrets.remove(RELAY_ISSUER_SECRET), - dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), - dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), - dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 7 }, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); + yield* Effect.all( + [ + dependencies.secrets.remove(CLOUD_LINKED_USER_ID), + dependencies.secrets.remove(RELAY_URL_SECRET), + dependencies.secrets.remove(RELAY_ISSUER_SECRET), + dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), + dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + ], + { concurrency: 7 }, + ); + yield* setCliDesiredCloudLink(false); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* setCliDesiredCloudLink(false); - return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }, Effect.catchIf( ServerSecretStore.isSecretStoreError, diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 08a639cd20c5..a32b349585d0 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -945,6 +945,7 @@ const buildAppUnderTest = (options?: { applyConfig: () => Effect.succeed({ status: "disabled" }), recoveryRequests: Stream.empty, requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, ...options?.layers?.cloudManagedEndpointRuntime, }), ), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8762d405cdc6..95b6f2fe1b15 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -4,6 +4,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -620,20 +621,26 @@ export const makeServerLayer = Layer.unwrap( if (typeof address === "string" || !("port" in address)) return; const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; - const recoverManagedTunnel = recoverManagedCloudTunnel(localOrigin).pipe( - Effect.retry({ - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const recoveryLock = yield* Semaphore.make(1); + const recoverManagedTunnel = recoveryLock.withPermits(1)( + recoverManagedCloudTunnel(localOrigin).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentCloudEndpointUnavailableError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), ), - Schedule.upTo({ duration: "10 minutes" }), + }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), - }), - Effect.tap((recovered) => - recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, - ), - Effect.catchCause((cause) => - Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ); yield* endpointRuntime.recoveryRequests.pipe( diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index c0811e82d923..536c4e289695 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -137,7 +137,7 @@ function testLayer(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { prepareDeprovision: () => Effect.succeed(null), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), provision: () => Effect.succeed({ @@ -243,6 +243,7 @@ describe("EnvironmentLinker", () => { deprovision: (input) => Effect.sync(() => { deprovisionedEnvironmentId = input.environmentId; + return true; }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index c4c06e9e723e..ba014dd8d576 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; @@ -17,6 +18,7 @@ describe("ManagedEndpointAllocations", () => { readonly updatedAt: string; } | undefined; + let condition: unknown; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); @@ -24,7 +26,12 @@ describe("ManagedEndpointAllocations", () => { set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { updated = values; return { - where: () => Effect.void, + where: (where: unknown) => { + condition = where; + return { + returning: () => Effect.succeed([{ environmentId: "environment-1" }]), + }; + }, }; }, }; @@ -33,13 +40,48 @@ describe("ManagedEndpointAllocations", () => { return Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; - yield* allocations.enableRecovery({ - userId: "user-1", - environmentId: "environment-1", - }); + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + environmentPublicKey: "public-key", + }), + ).toBe(true); expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); expect(updated?.recoveryEnabledAt).toBeDefined(); + const query = new PgDialect().sqlToQuery(condition as never); + expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); + expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); + expect(query.sql).toContain("for update"); + expect(query.params).toContain("tunnel-1"); + expect(query.params).toContain("public-key"); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("rejects recovery when the tunnel or active link no longer matches", () => { + const fakeDb = { + update: () => ({ + set: () => ({ + where: () => ({ + returning: () => Effect.succeed([]), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "missing-tunnel", + environmentPublicKey: "public-key", + }), + ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4fbdadd20239..6b67c6be93f1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,5 +1,6 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull } from "drizzle-orm"; +import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -8,7 +9,7 @@ import * as Schema from "effect/Schema"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; -import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface ManagedEndpointAllocation { readonly userId: string; @@ -99,6 +100,11 @@ interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey readonly updatedAt: string; } +interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly environmentPublicKey: string; +} + interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { readonly updatedAt: string; } @@ -126,8 +132,8 @@ export class ManagedEndpointAllocations extends Context.Service< input: ManagedEndpointAllocationKey, ) => Effect.Effect; readonly enableRecovery: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: EnableManagedEndpointRecoveryInput, + ) => Effect.Effect; readonly listByTunnelNames: ( tunnelNames: ReadonlyArray, ) => Effect.Effect< @@ -143,7 +149,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -328,16 +334,35 @@ export const make = Effect.gen(function* () { ); }), enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( - input: ManagedEndpointAllocationKey, + input: EnableManagedEndpointRecoveryInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ recoveryEnabledAt: now, updatedAt: now }) .where( - and(whereAllocation(input), isNull(relayManagedEndpointAllocations.recoveryEnabledAt)), + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + exists( + new QueryBuilder() + .select({ userId: relayEnvironmentLinks.userId }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .for("update"), + ), + ), ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -384,10 +409,11 @@ export const make = Effect.gen(function* () { claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { + const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) .set({ - updatedAt: DateTime.formatIso(yield* DateTime.now), + updatedAt: claimedAt, }) .where( and( @@ -411,7 +437,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed; + return claimed ? claimedAt : null; }), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 8ec3f3762b75..769466554c68 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -33,7 +33,7 @@ const config = RelayConfiguration.RelayConfiguration.of({ }); interface TunnelCall { - readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete"; + readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete"; readonly input: unknown; } @@ -62,6 +62,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId: function makeTunnelClient(calls: TunnelCall[] = []) { return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.sync(() => { + calls.push({ operation: "get", input: tunnelId }); + return { + id: tunnelId, + name: "managed-tunnel", + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }; + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -91,6 +101,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) { function makePersistentTunnelClient(calls: TunnelCall[] = []) { let tunnel: { readonly id: string; readonly name: string } | null = null; return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + calls.push({ operation: "get", input: tunnelId }); + return tunnel === null + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ) + : Effect.succeed({ + ...tunnel, + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }); + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -217,7 +244,12 @@ function makeAllocations(calls: AllocationCall[] = []) { }), enableRecovery: (input) => Effect.sync(() => { + const allocation = allocations.get(allocationKey(input)); + if (allocation?.tunnelId !== input.tunnelId) { + return false; + } recoveryEnabled.add(allocationKey(input)); + return true; }), listByTunnelNames: (tunnelNames) => Effect.sync(() => @@ -237,10 +269,10 @@ function makeAllocations(calls: AllocationCall[] = []) { allocation.tunnelId !== input.tunnelId || allocation.updatedAt !== input.updatedAt ) { - return false; + return null; } mutate(allocationKey(input), (current) => current); - return true; + return allocations.get(allocationKey(input))?.updatedAt ?? null; }), claimDeprovision: (input) => Effect.sync(() => { @@ -320,6 +352,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"): describe("ManagedEndpointProvider", () => { it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => { const tunnelClient = { + get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }), list: () => Effect.succeed({ result: [] }), create: (request: { readonly name: string }) => Effect.succeed({ id: "tunnel-id", name: request.name }), @@ -928,7 +961,7 @@ describe("ManagedEndpointProvider", () => { // longer matches what the release loaded, so the claim fails. const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - claimRelease: () => Effect.succeed(false), + claimRelease: () => Effect.succeed(null), }); const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated); @@ -974,6 +1007,80 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { + const tunnelCalls: TunnelCall[] = []; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makePersistentTunnelClient(tunnelCalls), + get: (tunnelId) => + Effect.succeed({ + id: tunnelId, + name: expectedManagedTunnelName("env_ABC"), + status: "healthy", + connsInactiveAt: null, + }), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel when a new provision replaces the release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedAt) => + claimedAt === null + ? Effect.void + : allocations.recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + }), + ), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 1789c014ae5d..41852f20f282 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare"; import * as Arr from "effect/Array"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Encoding from "effect/Encoding"; import * as Layer from "effect/Layer"; @@ -76,6 +77,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass Effect.Effect; + }) => Effect.Effect; /** * Deletes the provisioned Cloudflare tunnel while keeping the allocation * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per @@ -167,6 +169,8 @@ export class ManagedEndpointProvider extends Context.Service< readonly userId: string; readonly environmentId: string; readonly expectedTunnelId?: string; + readonly expectedInactiveBefore?: string; + readonly expectedStatus?: "inactive" | "down"; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} @@ -191,6 +195,7 @@ export interface ManagedEndpointTunnelListRequest { } const ManagedEndpointTunnelClientOperation = Schema.Literals([ + "get", "list", "create", "put-configuration", @@ -216,6 +221,9 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass Effect.Effect; readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< { readonly result: ReadonlyArray; @@ -352,17 +360,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean { ); } -function isNotFoundCause(cause: unknown): boolean { +export function isManagedEndpointNotFound(cause: unknown): boolean { if (typeof cause !== "object" || cause === null) { return false; } - if ("_tag" in cause && cause._tag === "NotFound") { + if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) { return true; } if ("status" in cause && cause.status === 404) { return true; } - return "cause" in cause && isNotFoundCause(cause.cause); + return "cause" in cause && isManagedEndpointNotFound(cause.cause); } type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError; @@ -374,9 +382,9 @@ const ignoreNotFound = ( Effect.asVoid, Effect.catchTags({ ManagedEndpointTunnelClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), }), ); @@ -418,7 +426,7 @@ export const make = Effect.gen(function* () { Effect.as(true), Effect.catchTags({ ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error), }), ); if (checkpointedRecordUpdated) { @@ -484,7 +492,7 @@ export const make = Effect.gen(function* () { const allocation = input.target === undefined ? yield* prepareDeprovision(input) : input.target; if (allocation === null) { - return; + return true; } const claimedAt = yield* allocations .claimDeprovision({ @@ -505,7 +513,7 @@ export const make = Effect.gen(function* () { ), ); if (claimedAt === null) { - return; + return false; } const dnsRecordId = allocation.dnsRecordId; if (dnsRecordId !== null) { @@ -535,7 +543,7 @@ export const make = Effect.gen(function* () { ), ); } - yield* allocations + return yield* allocations .removeClaimed({ userId: input.userId, environmentId: input.environmentId, @@ -583,7 +591,7 @@ export const make = Effect.gen(function* () { // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimed = yield* allocations + const claimedAt = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, @@ -601,9 +609,68 @@ export const make = Effect.gen(function* () { }), ), ); - if (!claimed) { + if (claimedAt === null) { return false; } + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const currentAllocation = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-allocation", + tunnelId, + cause, + }), + ), + ); + if ( + currentAllocation === null || + currentAllocation.tunnelId !== tunnelId || + currentAllocation.updatedAt !== claimedAt + ) { + return false; + } + + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTag("ManagedEndpointTunnelClientError", (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + ), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + input.expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== input.expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(input.expectedInactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + } yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => @@ -890,6 +957,18 @@ export const layerCloudflareBindings = ( Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ + get: (tunnelId) => + tunnelClient.get(tunnelId).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause, + }), + ), + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + ), list: (request) => tunnelClient.list(request).pipe( Effect.mapError( diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index c8e4816b348e..cc4c43b6ae23 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -55,24 +55,58 @@ function harness(input?: { readonly allocations?: ReadonlyArray; readonly namespace?: string; readonly failTunnelId?: string; + readonly missingOnDeleteTunnelId?: string; + readonly missingOnGetTunnelId?: string; + readonly reserveOnGetTunnelId?: string; + readonly refreshedTunnels?: ReadonlyMap; readonly skipTunnelId?: string; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; - const releases: Array<{ - readonly userId: string; - readonly environmentId: string; - readonly expectedTunnelId?: string; - }> = []; + const releases: Array< + Parameters[0] + > = []; + const remaining = [...(input?.tunnels ?? [])]; const recorded = (input?.allocations ?? []).map((entry) => { - const matching = input?.tunnels?.find((candidate) => candidate.id === entry.tunnelId); + const matching = remaining.find((candidate) => candidate.id === entry.tunnelId); return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; }); const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + if (tunnelId === input?.missingOnGetTunnelId) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + const found = + input?.refreshedTunnels?.get(tunnelId) ?? + remaining.find((candidate) => candidate.id === tunnelId); + if (found === undefined) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") { + recorded.push({ + ...allocation({ tunnelId, recoveryEnabled: false }), + tunnelName: found.name, + }); + } + return Effect.succeed(found); + }), list: (request) => Effect.sync(() => { listRequests.push(request); - const matching = (input?.tunnels ?? []).filter((entry) => entry.status === request.status); + const matching = remaining.filter((entry) => entry.status === request.status); const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); return { result: matching.slice(start, start + (request.perPage ?? 100)), @@ -87,16 +121,23 @@ function harness(input?: { putConfiguration: () => Effect.die("unused"), getToken: () => Effect.die("unused"), delete: (tunnelId) => - tunnelId === input?.failTunnelId + tunnelId === input?.failTunnelId || tunnelId === input?.missingOnDeleteTunnelId ? Effect.fail( new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ operation: "delete", tunnelId, - cause: "Cloudflare refused the deletion", + cause: + tunnelId === input?.missingOnDeleteTunnelId + ? { _tag: "NotFound" } + : "Cloudflare refused the deletion", }), ) : Effect.sync(() => { deleted.push(tunnelId); + const index = remaining.findIndex((candidate) => candidate.id === tunnelId); + if (index !== -1) { + remaining.splice(index, 1); + } }), }); const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -125,6 +166,12 @@ function harness(input?: { } if (request.expectedTunnelId !== undefined) { deleted.push(request.expectedTunnelId); + const index = remaining.findIndex( + (candidate) => candidate.id === request.expectedTunnelId, + ); + if (index !== -1) { + remaining.splice(index, 1); + } } return true; }), @@ -309,6 +356,81 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("keeps an orphan tunnel that reconnects before deletion", () => { + const listed = tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }); + const state = harness({ + tunnels: [listed], + refreshedTunnels: new Map([ + [ + "reconnected", + tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "healthy", + }), + ], + ]), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("treats an already deleted orphan tunnel as successfully removed", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "gone", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + missingOnDeleteTunnelId: "gone", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 1, + skippedLegacy: 0, + failed: 0, + }); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not delete an orphan tunnel reserved during the status check", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "reserved", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + reserveOnGetTunnelId: "reserved", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("does not count a tunnel that was replaced before its release", () => { const state = harness({ tunnels: [ @@ -397,6 +519,35 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("collects every page before deletions shift Cloudflare pagination", () => { + const entries = Array.from({ length: 120 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries + .slice(50, 100) + .map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: false })), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 120, + deleted: 70, + skippedLegacy: 50, + failed: 0, + }); + expect(state.deleted).toContain("tunnel-119"); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("limits each cleanup run to 100 tunnel deletions", () => { const state = harness({ tunnels: Array.from({ length: 105 }, (_, index) => diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index cfd933ebe650..da039f64030a 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -67,6 +67,42 @@ export const make = Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const deleteOrphan = Effect.fn("relay.managed_endpoint_reaper.delete_orphan")(function* (input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; + }) { + const current = yield* tunnels.get(input.tunnel.id).pipe( + Effect.map(Option.some), + Effect.catchTag("ManagedEndpointTunnelClientError", (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(Option.none()) + : Effect.fail(error), + ), + ); + if (Option.isNone(current)) { + return true; + } + if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { + return false; + } + if ((yield* allocations.listByTunnelNames([input.tunnel.name])).length > 0) { + return false; + } + return yield* tunnels.delete(input.tunnel.id).pipe( + Effect.as(true), + Effect.catchTag("ManagedEndpointTunnelClientError", (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(true) + : Effect.fail(error), + ), + ); + }); + const sweep = Effect.gen(function* () { const namespace = config.managedEndpointNamespace; if (!namespace) { @@ -77,14 +113,20 @@ export const make = Effect.gen(function* () { const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); const cutoffIso = DateTime.formatIso(cutoff); const prefix = managedEndpointTunnelNamePrefix(namespace); - let scanned = 0; let deleted = 0; let skippedLegacy = 0; let failed = 0; + const expired: Array<{ + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + }> = []; for (const status of ["down", "inactive"] as const) { let page = 1; - while (deleted < MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + while (true) { const response = yield* tunnels.list({ isDeleted: false, includePrefix: prefix, @@ -94,56 +136,13 @@ export const make = Effect.gen(function* () { page, perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, }); - const expired = response.result - .map((tunnel) => ({ tunnel, status, prefix, cutoff })) - .filter(isExpiredManagedTunnel) - .map(({ tunnel }) => tunnel); - scanned += expired.length; - - const recorded = yield* allocations.listByTunnelNames(expired.map((tunnel) => tunnel.name)); - const recordedByTunnelId = new Map( - recorded - .filter((allocation) => allocation.tunnelId !== null) - .map((allocation) => [allocation.tunnelId, allocation]), + expired.push( + ...response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => ({ tunnel, status })), ); - for (const tunnel of expired) { - if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { - break; - } - const allocation = recordedByTunnelId.get(tunnel.id); - if (allocation !== undefined && !allocation.recoveryEnabled) { - skippedLegacy += 1; - continue; - } - - const result = - allocation === undefined - ? yield* tunnels.delete(tunnel.id).pipe(Effect.as(true), Effect.result) - : yield* provider - .release({ - userId: allocation.userId, - environmentId: allocation.environmentId, - expectedTunnelId: tunnel.id, - }) - .pipe(Effect.result); - if (result._tag === "Failure") { - failed += 1; - yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - cause: result.failure, - }); - } else if (result.success) { - deleted += 1; - yield* Effect.logInfo("Deleted an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - status, - }); - } - } - const totalCount = response.resultInfo?.totalCount; if ( response.result.length === 0 || @@ -157,7 +156,54 @@ export const make = Effect.gen(function* () { } } - return { scanned, deleted, skippedLegacy, failed }; + const recorded = yield* allocations.listByTunnelNames(expired.map(({ tunnel }) => tunnel.name)); + const recordedByTunnelName = new Map( + recorded.map((allocation) => [allocation.tunnelName, allocation]), + ); + + for (const { tunnel, status } of expired) { + if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + break; + } + const allocation = recordedByTunnelName.get(tunnel.name); + if (allocation !== undefined && allocation.tunnelId !== tunnel.id) { + continue; + } + if (allocation !== undefined && !allocation.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + + const result = + allocation === undefined + ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) + : yield* provider + .release({ + userId: allocation.userId, + environmentId: allocation.environmentId, + expectedTunnelId: tunnel.id, + expectedInactiveBefore: cutoffIso, + expectedStatus: status, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + return { scanned: expired.length, deleted, skippedLegacy, failed }; }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); return ManagedEndpointReaper.of({ sweep }); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 12b39dd3e7e9..5453d492bb22 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -204,7 +204,7 @@ function relayUnlinkTestLayer(input?: { ManagedEndpointProvider.ManagedEndpointProvider.of({ provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.die("unused release"), }), ), @@ -225,8 +225,12 @@ const linkedEnvironmentRecord = { describe("relay managed tunnel recovery", () => { it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { - let recoveryEnabledFor: { readonly userId: string; readonly environmentId: string } | null = - null; + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; const runtime = { providerKind: "cloudflare_tunnel" as const, connectorToken: "replacement-token", @@ -248,6 +252,8 @@ describe("relay managed tunnel recovery", () => { expect(recoveryEnabledFor).toEqual({ userId: "user-1", environmentId: "environment-1", + tunnelId: "replacement-tunnel", + environmentPublicKey: "public-key", }); }).pipe( Effect.provide( @@ -264,6 +270,7 @@ describe("relay managed tunnel recovery", () => { enableRecovery: (input) => Effect.sync(() => { recoveryEnabledFor = input; + return true; }), }), ), @@ -365,15 +372,76 @@ describe("relay managed tunnel recovery", () => { wsBaseUrl: "wss://different.example.test/ws", providerKind: "cloudflare_tunnel", }, - runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "replacement-tunnel", + }, }), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ enableRecovery: () => Effect.sync(() => { recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("removes a recovered tunnel when its link disappears before registration", () => { + let lookups = 0; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "replacement-generation", + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; }), }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), ), ), ); @@ -473,6 +541,7 @@ describe("relay environment unlink", () => { Effect.sync(() => { expect(request.target).toBe(deprovisionTarget); calls.push("deprovision"); + return true; }), }), ), @@ -521,6 +590,7 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; }), }), ), @@ -548,6 +618,45 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; + }), + }), + ), + ); + }); + + it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => { + let lookups = 0; + const targets: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "tunnel-1", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "original-generation", + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + expect( + yield* unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + }), + ).toBe(true); + expect(targets).toEqual([target, undefined]); + }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + targets.push(captured ?? undefined); + return targets.length > 1; }), }), ), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 001e0de7e7d3..5f66dec9d6dc 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -456,11 +456,14 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a // retry to finish cleanup after an earlier Cloudflare failure. - yield* managedEndpointProvider.deprovision({ + const deprovisioned = yield* managedEndpointProvider.deprovision({ userId: input.userId, environmentId: input.environmentId, target: deprovisionTarget, }); + if (!deprovisioned && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision(input); + } return unlinked; }, ); @@ -493,17 +496,36 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( environmentId: input.environmentId, origin: input.origin, }); + const recoveredTunnelId = recovered.runtime.tunnelId; if ( + recoveredTunnelId === undefined || recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { return yield* new HttpApiError.Unauthorized({}); } - yield* allocations.enableRecovery({ + const enabled = yield* allocations.enableRecovery({ userId: input.userId, environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + environmentPublicKey: input.environmentPublicKey, }); + if (!enabled) { + const target = yield* managedEndpointProvider.prepareDeprovision(input); + if (target !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel after its link was removed", { + userId: input.userId, + environmentId: input.environmentId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } return { endpoint: recovered.endpoint, endpointRuntime: recovered.runtime, @@ -1053,6 +1075,8 @@ export const serverApi = HttpApiBuilder.group( relayInternalErrorResponse("upstream_unavailable"), ManagedEndpointProvisioningFailed: () => relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), }), mapRelayCommonApiErrors("not_authorized"), From a86c6d113c91d02d7261717f7fb171034b931771 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 20:50:44 -0700 Subject: [PATCH 03/10] fix(connect): serialize tunnel deletion and recovery ownership --- .../src/cloud/ManagedEndpointRuntime.test.ts | 24 +++ .../src/cloud/ManagedEndpointRuntime.ts | 15 ++ apps/server/src/cloud/http.ts | 96 ++++----- apps/server/src/server.ts | 7 +- .../migration.sql | 1 - .../migration.sql | 2 + .../snapshot.json | 15 +- .../environments/EnvironmentConnector.test.ts | 3 + .../ManagedEndpointAllocations.test.ts | 58 +++++- .../ManagedEndpointAllocations.ts | 118 +++++++++--- .../ManagedEndpointProvider.test.ts | 60 +++++- .../environments/ManagedEndpointProvider.ts | 182 +++++++++++------- .../ManagedEndpointReaper.test.ts | 29 ++- .../src/environments/ManagedEndpointReaper.ts | 43 +++-- infra/relay/src/http/Api.test.ts | 5 +- infra/relay/src/http/Api.ts | 12 +- infra/relay/src/persistence/schema.ts | 1 + infra/relay/src/worker.ts | 4 +- 18 files changed, 486 insertions(+), 189 deletions(-) delete mode 100644 infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql create mode 100644 infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql rename infra/relay/migrations/postgres/{20260825010804_managed_endpoint_recovery => 20260825034308_managed_endpoint_recovery}/snapshot.json (99%) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 54a12373c732..4dc9cc2ca7b0 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -80,6 +80,30 @@ function makeHandle(input: { } describe("CloudManagedEndpointRuntime", () => { + it("retries connector startup failures but stops for unsupported runtimes", () => { + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "The relay client is not installed.", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "spawn failed", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + reason: "Relay client is unsupported on linux-arm.", + }), + ).toBe(false); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }), + ).toBe(false); + }); + it.effect("serializes updates to persisted cloud link state", () => Effect.gen(function* () { const firstEntered = yield* Deferred.make(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index cf566ac0a4c8..44ef29ecf8a3 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -82,6 +82,21 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +/** Connector startup failures can clear after installation or a later spawn attempt. */ +export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { + if (typeof status !== "object" || status === null || !("status" in status)) { + return false; + } + if (status.status !== "failed") { + return false; + } + return !( + "reason" in status && + typeof status.reason === "string" && + status.reason.startsWith("Relay client is unsupported on ") + ); +} + function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { return JSON.stringify({ providerKind: config.providerKind, diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 9ae34e6d0d5c..14d913cd105c 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -454,57 +454,56 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, - options?: { readonly requestRecovery?: boolean }, + options?: { readonly requestRecovery?: boolean; readonly lockHeld?: boolean }, ) { - return yield* dependencies.endpointRuntime.withLinkStateLock( - Effect.gen(function* () { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, + const apply = Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, + }); + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + const ok = + endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; + if (!ok) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, - }); - } + } - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), - ); + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); + yield* dependencies.secrets.set( + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), + ); + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); yield* dependencies.secrets.set( - CLOUD_MINT_PUBLIC_KEY, - stringToBytes(payload.cloudMintPublicKey), + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), ); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); - yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), - ); - if (options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if (options?.requestRecovery !== false) { + yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; - }), - ); + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }); + return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply); }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( @@ -629,7 +628,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi cloudMintPublicKey: link.cloudMintPublicKey, endpointRuntime: link.endpointRuntime, }, - { requestRecovery: false }, + { requestRecovery: false, lockHeld: true }, ); }, Effect.catchIf( @@ -647,7 +646,10 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")( function* (localOrigin: string) { - return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin); + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + reconcileDesiredCloudLinkWith(dependencies, localOrigin), + ); }, ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 95b6f2fe1b15..2af1b9981c79 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -627,7 +627,10 @@ export const makeServerLayer = Layer.unwrap( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentCloudEndpointUnavailableError", + (error._tag !== "EnvironmentCloudEndpointUnavailableError" || + CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + )), schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), @@ -638,7 +641,7 @@ export const makeServerLayer = Layer.unwrap( Effect.tap((recovered) => recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), ), ), diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql deleted file mode 100644 index f528c09fbc7f..000000000000 --- a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/migration.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64); \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql new file mode 100644 index 000000000000..4da6b524efa4 --- /dev/null +++ b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json similarity index 99% rename from infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json rename to infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json index 648638a0e17d..21afe569ac4e 100644 --- a/infra/relay/migrations/postgres/20260825010804_managed_endpoint_recovery/snapshot.json +++ b/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "f3c6f2a5-2ecf-43cb-b381-98790fdca66e", + "id": "a0128e5a-4bba-4f2d-9851-82c3744dae2c", "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], "ddl": [ { @@ -877,6 +877,19 @@ "schema": "public", "table": "relay_managed_endpoint_allocations" }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, { "type": "varchar(64)", "typeSchema": null, diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index dfa809a9f151..31e6d27c8fcd 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -189,6 +189,7 @@ function makeAllocations( dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -200,6 +201,7 @@ function makeAllocations( enableRecovery: () => Effect.die("unused"), listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -474,6 +476,7 @@ describe("EnvironmentConnector", () => { dnsRecordId: "dns-record-id", readyAt: null, updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ba014dd8d576..73612590e5fa 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; @@ -93,6 +94,7 @@ describe("ManagedEndpointAllocations", () => { dnsRecordId: "dns-1", readyAt: "2026-08-25T12:00:00.000Z", updatedAt: "2026-08-25T12:00:00.000Z", + generation: 1, }; const fakeDb = { select: () => ({ @@ -138,16 +140,14 @@ describe("ManagedEndpointAllocations", () => { ); it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { - let claimedAt: string | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly updatedAt: string }) => { - claimedAt = values.updatedAt; + set: (_values: { readonly updatedAt: string }) => { return { where: () => ({ - returning: () => Effect.succeed([{ userId: "user-1" }]), + returning: () => Effect.succeed([{ generation: 8 }]), }), }; }, @@ -160,14 +160,58 @@ describe("ManagedEndpointAllocations", () => { const generation = yield* allocations.claimDeprovision({ userId: "user-1", environmentId: "environment-1", - updatedAt: "captured-generation", + generation: 7, }); - expect(generation).toBe(claimedAt); + expect(generation).toBe(8); expect(generation).not.toBeNull(); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); + it.effect("holds the claimed allocation row while deleting its tunnel", () => { + const operations: string[] = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => { + operations.push("transaction"); + }).pipe(Effect.andThen(effect)), + }, + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => ({ + for: (strength: string) => + Effect.sync(() => { + operations.push(`lock:${strength}`); + return [{ generation: 7 }]; + }), + }), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.withClaimedTunnel( + { + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + generation: 7, + }, + Effect.sync(() => { + operations.push("delete"); + return true; + }), + ); + + expect(Option.getOrNull(result)).toBe(true); + expect(operations).toEqual(["transaction", "lock:update", "delete"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("does not remove an allocation superseded after a deprovision claim", () => { const fakeDb = { delete: (table: unknown) => { @@ -186,7 +230,7 @@ describe("ManagedEndpointAllocations", () => { yield* allocations.removeClaimed({ userId: "user-1", environmentId: "environment-1", - updatedAt: "outdated-claim-generation", + generation: 7, }), ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 6b67c6be93f1..4e5edd1b1bb0 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,11 +1,13 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq, exists, inArray, isNull } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm"; import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import { isSqlError } from "effect/unstable/sql/SqlError"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; @@ -19,11 +21,8 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; - /** - * Doubles as the allocation's generation marker: every mutation rewrites it, - * so `claimRelease` can detect a provision that raced a release. - */ readonly updatedAt: string; + readonly generation: number; } export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { @@ -57,6 +56,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "mark-ready", "enable-recovery", "list-tunnels", + "lock-tunnel", "claim-release", "claim-deprovision", "remove", @@ -89,6 +89,7 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; + readonly generation: number; } interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { @@ -97,7 +98,7 @@ interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; - readonly updatedAt: string; + readonly generation: number; } interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { @@ -106,11 +107,11 @@ interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKe } interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } export class ManagedEndpointAllocations extends Context.Service< @@ -124,7 +125,7 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, ) => Effect.Effect; @@ -143,13 +144,17 @@ export class ManagedEndpointAllocations extends Context.Service< /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the - * caller loaded. A concurrent provision rewrites `updatedAt` when it + * caller loaded. A concurrent provision increments `generation` when it * records its tunnel, which makes a stale claim fail and keeps the freshly * issued tunnel alive. */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; + readonly withClaimedTunnel: ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ) => Effect.Effect, E | ManagedEndpointAllocationPersistenceError, R>; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -159,7 +164,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimDeprovision: ( input: ClaimManagedEndpointDeprovisionInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly remove: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; @@ -178,6 +183,7 @@ const allocationSelection = { dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, updatedAt: relayManagedEndpointAllocations.updatedAt, + generation: relayManagedEndpointAllocations.generation, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -269,14 +275,22 @@ export const make = Effect.gen(function* () { recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* ( input: RecordManagedEndpointTunnelInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -296,6 +310,7 @@ export const make = Effect.gen(function* () { .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where(whereAllocation(input)) .pipe( @@ -319,6 +334,7 @@ export const make = Effect.gen(function* () { .set({ readyAt: now, updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where(whereAllocation(input)) .pipe( @@ -339,7 +355,11 @@ export const make = Effect.gen(function* () { const now = DateTime.formatIso(yield* DateTime.now); return yield* db .update(relayManagedEndpointAllocations) - .set({ recoveryEnabledAt: now, updatedAt: now }) + .set({ + recoveryEnabledAt: now, + updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), @@ -409,22 +429,22 @@ export const make = Effect.gen(function* () { claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) .set({ - updatedAt: claimedAt, + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where( and( whereAllocation(input), eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -437,24 +457,66 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), + withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")( + function* ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ): Effect.fn.Return, E | ManagedEndpointAllocationPersistenceError, R> { + const lockError = (cause: unknown) => + new ManagedEndpointAllocationPersistenceError({ + operation: "lock-tunnel", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + cause, + }); + return yield* db.$client + .withTransaction( + db + .select({ generation: relayManagedEndpointAllocations.generation }) + .from(relayManagedEndpointAllocations) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .limit(1) + .for("update") + .pipe( + Effect.mapError(lockError), + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed(Option.none()) + : effect.pipe(Effect.map(Option.some)), + ), + ), + ) + .pipe(Effect.mapError((cause) => (isSqlError(cause) ? lockError(cause) : cause))); + }, + ), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) - .set({ updatedAt: claimedAt }) + .set({ + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -466,7 +528,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* ( input: ManagedEndpointAllocationKey, @@ -494,7 +556,7 @@ export const make = Effect.gen(function* () { .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) .returning({ userId: relayManagedEndpointAllocations.userId }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 769466554c68..e1168543d393 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -6,6 +6,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as RelayConfiguration from "../Config.ts"; @@ -196,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) { ) => { const allocation = allocations.get(key); if (allocation !== undefined) { - allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` }); + allocations.set(key, { + ...change(allocation), + generation: allocation.generation + 1, + updatedAt: `generation-${++generation}`, + }); } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -214,6 +219,7 @@ function makeAllocations(calls: AllocationCall[] = []) { dnsRecordId: null, readyAt: null, updatedAt: `generation-${++generation}`, + generation: 0, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -221,10 +227,15 @@ function makeAllocations(calls: AllocationCall[] = []) { recordTunnel: (input) => Effect.sync(() => { calls.push({ operation: "recordTunnel", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, })); + return true; }), recordDns: (input) => Effect.sync(() => { @@ -267,22 +278,29 @@ function makeAllocations(calls: AllocationCall[] = []) { if ( allocation === undefined || allocation.tunnelId !== input.tunnelId || - allocation.updatedAt !== input.updatedAt + allocation.generation !== input.generation ) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; + }), + withClaimedTunnel: (input, effect) => + Effect.suspend(() => { + const current = allocations.get(allocationKey(input)); + return current?.tunnelId === input.tunnelId && current.generation === input.generation + ? effect.pipe(Effect.map(Option.some)) + : Effect.succeed(Option.none()); }), claimDeprovision: (input) => Effect.sync(() => { calls.push({ operation: "claimDeprovision", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; }), remove: (input) => Effect.sync(() => { @@ -293,7 +311,7 @@ function makeAllocations(calls: AllocationCall[] = []) { Effect.sync(() => { calls.push({ operation: "removeClaimed", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return false; } allocations.delete(allocationKey(input)); @@ -1007,6 +1025,31 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("rejects a tunnel recorded after its allocation generation changed", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordTunnel: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-tunnel", + }); + }).pipe(Effect.provide(layer)); + }); + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { const tunnelCalls: TunnelCall[] = []; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ @@ -1048,13 +1091,14 @@ describe("ManagedEndpointProvider", () => { ...allocations, claimRelease: (input) => allocations.claimRelease(input).pipe( - Effect.tap((claimedAt) => - claimedAt === null + Effect.tap((claimedGeneration) => + claimedGeneration === null ? Effect.void : allocations.recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: "replacement-tunnel", + generation: claimedGeneration, }), ), ), diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 41852f20f282..f61a853f52bf 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -494,11 +494,11 @@ export const make = Effect.gen(function* () { if (allocation === null) { return true; } - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimDeprovision({ userId: input.userId, environmentId: input.environmentId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -512,7 +512,7 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { + if (claimedGeneration === null) { return false; } const dnsRecordId = allocation.dnsRecordId; @@ -547,7 +547,7 @@ export const make = Effect.gen(function* () { .removeClaimed({ userId: input.userId, environmentId: input.environmentId, - updatedAt: claimedAt, + generation: claimedGeneration, }) .pipe( Effect.mapError( @@ -586,17 +586,17 @@ export const make = Effect.gen(function* () { } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment - // restart) rewrites updatedAt when it records its tunnel, so a stale + // restart) increments the generation when it records its tunnel, so a stale // claim means the recorded tunnel may already back a fresh connector and // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, tunnelId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -609,69 +609,10 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { + if (claimedGeneration === null) { return false; } - if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { - const currentAllocation = yield* allocations.get(input).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-allocation", - tunnelId, - cause, - }), - ), - ); - if ( - currentAllocation === null || - currentAllocation.tunnelId !== tunnelId || - currentAllocation.updatedAt !== claimedAt - ) { - return false; - } - - const currentTunnel = yield* tunnels.get(tunnelId).pipe( - Effect.map(Option.some), - Effect.catchTag("ManagedEndpointTunnelClientError", (cause) => - isManagedEndpointNotFound(cause.cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-tunnel", - tunnelId, - cause, - }), - ), - ), - ); - if (Option.isNone(currentTunnel)) { - return true; - } - const inactiveAt = - input.expectedStatus === "down" - ? currentTunnel.value.connsInactiveAt - : currentTunnel.value.createdAt; - if ( - currentTunnel.value.id !== tunnelId || - currentTunnel.value.status !== input.expectedStatus || - typeof inactiveAt !== "string" - ) { - return false; - } - const inactiveTime = DateTime.make(inactiveAt); - const cutoff = DateTime.make(input.expectedInactiveBefore); - if ( - Option.isNone(inactiveTime) || - Option.isNone(cutoff) || - inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds - ) { - return false; - } - } - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => new ManagedEndpointDeprovisioningFailed({ @@ -682,6 +623,97 @@ export const make = Effect.gen(function* () { }), ), ); + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const released = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + Effect.gen(function* () { + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + input.expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== input.expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(input.expectedInactiveBefore!); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + + const finalGeneration = yield* allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (finalGeneration === null) { + return false; + } + yield* deleteTunnel; + return true; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + }), + ); + return Option.getOrElse(released, () => false); + } + yield* deleteTunnel; // The recorded tunnelId is now stale, but the allocation row is left // untouched deliberately: connect/status authorization requires a fully // recorded allocation, and an offline environment must keep reporting @@ -806,11 +838,12 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - yield* allocations + const recordedTunnel = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: tunnel.id, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -826,6 +859,17 @@ export const make = Effect.gen(function* () { }), ), ); + if (!recordedTunnel) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed during provisioning.", + }); + } yield* tunnels .putConfiguration(tunnel.id, { diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index cc4c43b6ae23..335d58c3c217 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -34,18 +34,19 @@ function tunnel(input: { } function allocation(input: { - readonly tunnelId: string; + readonly tunnelId: string | null; readonly recoveryEnabled: boolean; }): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { return { userId: "user-1", - environmentId: `environment-${input.tunnelId}`, - hostname: `${input.tunnelId}.example.test`, + environmentId: `environment-${input.tunnelId ?? "pending"}`, + hostname: `${input.tunnelId ?? "pending"}.example.test`, tunnelId: input.tunnelId, tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, dnsRecordId: "dns-1", readyAt: "2026-08-25T11:00:00.000Z", updatedAt: "2026-08-25T11:00:00.000Z", + generation: 1, recoveryEnabled: input.recoveryEnabled, }; } @@ -150,6 +151,7 @@ function harness(input?: { listByTunnelNames: (tunnelNames) => Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -356,6 +358,27 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("removes an expired tunnel that was created but never recorded", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "unrecorded", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: null, recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["unrecorded"]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("keeps an orphan tunnel that reconnects before deletion", () => { const listed = tunnel({ id: "reconnected", diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index da039f64030a..0442d5e58a80 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -78,11 +78,12 @@ export const make = Effect.gen(function* () { }) { const current = yield* tunnels.get(input.tunnel.id).pipe( Effect.map(Option.some), - Effect.catchTag("ManagedEndpointTunnelClientError", (error) => - ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) - ? Effect.succeed(Option.none()) - : Effect.fail(error), - ), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(Option.none()) + : Effect.fail(error), + }), ); if (Option.isNone(current)) { return true; @@ -90,16 +91,21 @@ export const make = Effect.gen(function* () { if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { return false; } - if ((yield* allocations.listByTunnelNames([input.tunnel.name])).length > 0) { + if ( + (yield* allocations.listByTunnelNames([input.tunnel.name])).some( + (allocation) => allocation.tunnelId !== null, + ) + ) { return false; } return yield* tunnels.delete(input.tunnel.id).pipe( Effect.as(true), - Effect.catchTag("ManagedEndpointTunnelClientError", (error) => - ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) - ? Effect.succeed(true) - : Effect.fail(error), - ), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(true) + : Effect.fail(error), + }), ); }); @@ -166,21 +172,26 @@ export const make = Effect.gen(function* () { break; } const allocation = recordedByTunnelName.get(tunnel.name); - if (allocation !== undefined && allocation.tunnelId !== tunnel.id) { + if ( + allocation !== undefined && + allocation.tunnelId !== null && + allocation.tunnelId !== tunnel.id + ) { continue; } - if (allocation !== undefined && !allocation.recoveryEnabled) { + const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; + if (owner !== undefined && !owner.recoveryEnabled) { skippedLegacy += 1; continue; } const result = - allocation === undefined + owner === undefined ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) : yield* provider .release({ - userId: allocation.userId, - environmentId: allocation.environmentId, + userId: owner.userId, + environmentId: owner.environmentId, expectedTunnelId: tunnel.id, expectedInactiveBefore: cutoffIso, expectedStatus: status, diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 5453d492bb22..f759c9166e2e 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -403,6 +403,7 @@ describe("relay managed tunnel recovery", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "replacement-generation", + generation: 3, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -493,6 +494,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "generation-before-unlink", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -637,6 +639,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "original-generation", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -646,7 +649,7 @@ describe("relay environment unlink", () => { environmentId: "environment-1", }), ).toBe(true); - expect(targets).toEqual([target, undefined]); + expect(targets).toEqual([target, target]); }).pipe( Effect.provide( relayUnlinkTestLayer({ diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5f66dec9d6dc..58aefda36b60 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -461,8 +461,11 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron environmentId: input.environmentId, target: deprovisionTarget, }); - if (!deprovisioned && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision(input); + if (!deprovisioned) { + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); + if (retryTarget !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + } } return unlinked; }, @@ -512,9 +515,10 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( environmentPublicKey: input.environmentPublicKey, }); if (!enabled) { - const target = yield* managedEndpointProvider.prepareDeprovision(input); + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); if (target !== null && (yield* links.getForUser(input)) === null) { - yield* managedEndpointProvider.deprovision({ ...input, target }).pipe( + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( Effect.catch((cause) => Effect.logWarning("Failed to clean up a tunnel after its link was removed", { userId: input.userId, diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index ddd276ccfd70..a53be1ea1a30 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -94,6 +94,7 @@ export const relayManagedEndpointAllocations = pgTable( dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 2f09fd3e5b9c..b9f5a947957f 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -278,7 +278,7 @@ export const ApiLive = Api.make( ), ), ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), @@ -289,7 +289,7 @@ export const ApiLive = Api.make( ? Effect.logInfo("Finished managed tunnel cleanup", result) : Effect.void, ), - Effect.catchCause((cause) => + Effect.catch((cause) => Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), ), ), From 4e997da89c37ccd601ee2bdfe6965399a9baa052 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:09:09 -0700 Subject: [PATCH 04/10] fix(connect): keep tunnel setup and teardown in sync --- .../src/cloud/ManagedEndpointRuntime.test.ts | 4 + .../src/cloud/ManagedEndpointRuntime.ts | 12 +- apps/server/src/server.test.ts | 1 + apps/server/src/server.ts | 15 +- .../ManagedEndpointAllocations.ts | 45 ++- .../ManagedEndpointProvider.test.ts | 84 +++++- .../environments/ManagedEndpointProvider.ts | 272 ++++++++++++------ infra/relay/src/worker.ts | 13 +- 8 files changed, 319 insertions(+), 127 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 4dc9cc2ca7b0..40f8a271dd5a 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -84,18 +84,21 @@ describe("CloudManagedEndpointRuntime", () => { expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "not-installed", reason: "The relay client is not installed.", }), ).toBe(true); expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "spawn-failed", reason: "spawn failed", }), ).toBe(true); expect( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "failed", + failure: "unsupported-platform", reason: "Relay client is unsupported on linux-arm.", }), ).toBe(false); @@ -442,6 +445,7 @@ describe("CloudManagedEndpointRuntime", () => { expect(status).toEqual({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "The relay client is not installed.", }); expect(spawn).not.toHaveBeenCalled(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 44ef29ecf8a3..ca12b430649f 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -37,6 +37,7 @@ export type CloudManagedEndpointRuntimeStatus = | { readonly status: "failed"; readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; + readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed"; readonly reason: string; readonly tunnelId?: string; readonly tunnelName?: string; @@ -87,14 +88,10 @@ export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolea if (typeof status !== "object" || status === null || !("status" in status)) { return false; } - if (status.status !== "failed") { + if (status.status !== "failed" || !("failure" in status)) { return false; } - return !( - "reason" in status && - typeof status.reason === "string" && - status.reason.startsWith("Relay client is unsupported on ") - ); + return status.failure === "not-installed" || status.failure === "spawn-failed"; } function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { @@ -236,6 +233,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed", reason: executable.status === "unsupported" ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.` @@ -278,6 +276,7 @@ export const make = Effect.gen(function* () { Effect.as({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: String(cause), ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -312,6 +311,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: "Relay client did not start.", ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index a32b349585d0..13867bf25d74 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -2934,6 +2934,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { Effect.succeed({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "cloudflared missing", tunnelId: "tunnel-1", }), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 2af1b9981c79..ddbe78b44ec2 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,4 +1,5 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -584,10 +585,6 @@ export const makeServerLayer = Layer.unwrap( : Layer.empty; const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( Effect.gen(function* () { - if (!hasCloudPublicConfig) { - yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); - return; - } const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( Effect.timeout("10 seconds"), Effect.tap((released) => @@ -641,8 +638,12 @@ export const makeServerLayer = Layer.unwrap( Effect.tap((recovered) => recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, ), - Effect.catch((cause) => - Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { + cause, + }), ), ), ); @@ -656,7 +657,7 @@ export const makeServerLayer = Layer.unwrap( // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - if (yield* CloudCliState.readCliDesiredCloudLink) { + if (hasCloudPublicConfig && (yield* CloudCliState.readCliDesiredCloudLink)) { yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4e5edd1b1bb0..afd77151a34a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -94,6 +94,13 @@ interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { readonly dnsRecordId: string; + readonly tunnelId: string; + readonly generation: number; +} + +interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly generation: number; } interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { @@ -125,13 +132,13 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly markReady: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: MarkManagedEndpointReadyInput, + ) => Effect.Effect; readonly enableRecovery: ( input: EnableManagedEndpointRecoveryInput, ) => Effect.Effect; @@ -288,9 +295,9 @@ export const make = Effect.gen(function* () { eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -305,15 +312,23 @@ export const make = Effect.gen(function* () { recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* ( input: RecordManagedEndpointDnsInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -326,18 +341,26 @@ export const make = Effect.gen(function* () { ); }), markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* ( - input: ManagedEndpointAllocationKey, + input: MarkManagedEndpointReadyInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ readyAt: now, updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index e1168543d393..69d208a276ba 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -229,29 +229,39 @@ function makeAllocations(calls: AllocationCall[] = []) { calls.push({ operation: "recordTunnel", input }); const current = allocations.get(allocationKey(input)); if (current?.generation !== input.generation) { - return false; + return null; } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, })); - return true; + return allocations.get(allocationKey(input))?.generation ?? null; }), recordDns: (input) => Effect.sync(() => { calls.push({ operation: "recordDns", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, dnsRecordId: input.dnsRecordId, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), markReady: (input) => Effect.sync(() => { calls.push({ operation: "markReady", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, readyAt: "2026-06-02T00:00:00.000Z", })); + return true; }), enableRecovery: (input) => Effect.sync(() => { @@ -1026,12 +1036,13 @@ describe("ManagedEndpointProvider", () => { }); it.effect("rejects a tunnel recorded after its allocation generation changed", () => { + const tunnelCalls: TunnelCall[] = []; const allocations = makeAllocations(); const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - recordTunnel: () => Effect.succeed(false), + recordTunnel: () => Effect.succeed(null), }); - const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -1047,6 +1058,57 @@ describe("ManagedEndpointProvider", () => { _tag: "ManagedEndpointProvisioningFailed", stage: "record-tunnel", }); + expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create", "delete"]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordDns: () => Effect.succeed(null), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not mark a superseded tunnel allocation as ready", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + markReady: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "mark-allocation-ready", + }); }).pipe(Effect.provide(layer)); }); @@ -1094,12 +1156,14 @@ describe("ManagedEndpointProvider", () => { Effect.tap((claimedGeneration) => claimedGeneration === null ? Effect.void - : allocations.recordTunnel({ - userId: input.userId, - environmentId: input.environmentId, - tunnelId: "replacement-tunnel", - generation: claimedGeneration, - }), + : allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid), ), ), }); diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index f61a853f52bf..b3752a3b994e 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -515,52 +515,83 @@ export const make = Effect.gen(function* () { if (claimedGeneration === null) { return false; } - const dnsRecordId = allocation.dnsRecordId; - if (dnsRecordId !== null) { - yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-dns-record", - dnsRecordId, - cause, - }), - ), - ); - } const tunnelId = allocation.tunnelId; - if (tunnelId !== null) { - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-tunnel", - tunnelId, - cause, - }), - ), - ); + const deprovision = Effect.gen(function* () { + const dnsRecordId = allocation.dnsRecordId; + if (dnsRecordId !== null) { + yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-dns-record", + dnsRecordId, + cause, + }), + ), + ); + } + if (tunnelId !== null) { + yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-tunnel", + tunnelId, + cause, + }), + ), + ); + } + return yield* allocations + .removeClaimed({ + userId: input.userId, + environmentId: input.environmentId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "remove-allocation", + ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + }); + if (tunnelId === null) { + return yield* deprovision; } - return yield* allocations - .removeClaimed({ - userId: input.userId, - environmentId: input.environmentId, - generation: claimedGeneration, - }) + const removed = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + deprovision, + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "remove-allocation", - ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-deprovision", + tunnelId, + cause, + }), + ), + }), ); + return Option.getOrElse(removed, () => false); }), release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -624,6 +655,48 @@ export const make = Effect.gen(function* () { ), ); if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const expectedStatus = input.expectedStatus; + const inactiveBefore = input.expectedInactiveBefore; + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(inactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + const released = yield* allocations .withClaimedTunnel( { @@ -633,46 +706,6 @@ export const make = Effect.gen(function* () { generation: claimedGeneration, }, Effect.gen(function* () { - const currentTunnel = yield* tunnels.get(tunnelId).pipe( - Effect.map(Option.some), - Effect.catchTags({ - ManagedEndpointTunnelClientError: (cause) => - isManagedEndpointNotFound(cause.cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "load-tunnel", - tunnelId, - cause, - }), - ), - }), - ); - if (Option.isNone(currentTunnel)) { - return true; - } - const inactiveAt = - input.expectedStatus === "down" - ? currentTunnel.value.connsInactiveAt - : currentTunnel.value.createdAt; - if ( - currentTunnel.value.id !== tunnelId || - currentTunnel.value.status !== input.expectedStatus || - typeof inactiveAt !== "string" - ) { - return false; - } - const inactiveTime = DateTime.make(inactiveAt); - const cutoff = DateTime.make(input.expectedInactiveBefore!); - if ( - Option.isNone(inactiveTime) || - Option.isNone(cutoff) || - inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds - ) { - return false; - } - const finalGeneration = yield* allocations .claimRelease({ userId: input.userId, @@ -694,6 +727,8 @@ export const make = Effect.gen(function* () { if (finalGeneration === null) { return false; } + // Keep only the final delete inside the row lock so a concurrent + // provision cannot record this tunnel while Cloudflare removes it. yield* deleteTunnel; return true; }), @@ -805,13 +840,16 @@ export const make = Effect.gen(function* () { ); const { hostname, tunnelName } = allocation; - const tunnelResponse = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( + const selectedTunnel = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( Effect.map((tunnels) => tunnels.result), Effect.map(Arr.findFirst((tunnel) => tunnel.name === tunnelName)), Effect.flatMap( Option.match({ - onSome: (tunnel) => Effect.succeed(tunnel), - onNone: () => tunnels.create({ name: tunnelName, configSrc: "cloudflare" }), + onSome: (tunnel) => Effect.succeed({ tunnel, created: false }), + onNone: () => + tunnels + .create({ name: tunnelName, configSrc: "cloudflare" }) + .pipe(Effect.map((tunnel) => ({ tunnel, created: true }))), }), ), Effect.mapError( @@ -826,6 +864,7 @@ export const make = Effect.gen(function* () { }), ), ); + const tunnelResponse = selectedTunnel.tunnel; if (!tunnelResponse.id || tunnelResponse.name !== tunnelName) { return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, @@ -838,7 +877,7 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - const recordedTunnel = yield* allocations + const tunnelGeneration = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, @@ -859,7 +898,34 @@ export const make = Effect.gen(function* () { }), ), ); - if (!recordedTunnel) { + if (tunnelGeneration === null) { + if (selectedTunnel.created) { + const current = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ); + if (current?.tunnelId !== tunnel.id) { + yield* ignoreNotFound(tunnels.delete(tunnel.id)).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to remove a tunnel that lost its allocation", { + tunnelId: tunnel.id, + tunnelName, + cause, + }), + ), + ); + } + } return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, @@ -919,11 +985,13 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const dnsGeneration = yield* allocations .recordDns({ userId: input.userId, environmentId: input.environmentId, dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, }) .pipe( Effect.mapError( @@ -940,6 +1008,18 @@ export const make = Effect.gen(function* () { }), ), ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause: "The tunnel allocation changed before its DNS record was saved.", + }); + } const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( @@ -956,10 +1036,12 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const ready = yield* allocations .markReady({ userId: input.userId, environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: dnsGeneration, }) .pipe( Effect.mapError( @@ -976,6 +1058,18 @@ export const make = Effect.gen(function* () { }), ), ); + if (!ready) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "mark-allocation-ready", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause: "The tunnel allocation changed before it became ready.", + }); + } return { endpoint: managedEndpointForHostname(hostname), diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index b9f5a947957f..fe9c085fdea8 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -2,6 +2,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle"; import * as Config from "effect/Config"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -278,8 +279,10 @@ export const ApiLive = Api.make( ), ), ), - Effect.catch((cause) => - Effect.logWarning("Failed to prune expired relay state", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), ManagedEndpointReaper.ManagedEndpointReaper.pipe( @@ -289,8 +292,10 @@ export const ApiLive = Api.make( ? Effect.logInfo("Finished managed tunnel cleanup", result) : Effect.void, ), - Effect.catch((cause) => - Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), ), ), ], From 9f991d937c6a7505a4714d2086798422f87a2dd6 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:23:53 -0700 Subject: [PATCH 05/10] fix(connect): guard tunnel configuration and batch cleanup --- apps/server/src/server.ts | 11 +- .../ManagedEndpointAllocations.test.ts | 21 ++ .../ManagedEndpointAllocations.ts | 64 +++--- .../ManagedEndpointProvider.test.ts | 58 ++++++ .../environments/ManagedEndpointProvider.ts | 190 +++++++++++++----- 5 files changed, 265 insertions(+), 79 deletions(-) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index ddbe78b44ec2..0ab72658cff3 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -657,7 +657,16 @@ export const makeServerLayer = Layer.unwrap( // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - if (hasCloudPublicConfig && (yield* CloudCliState.readCliDesiredCloudLink)) { + const wantsCliLink = hasCloudPublicConfig + ? yield* CloudCliState.readCliDesiredCloudLink.pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to read the desired T3 Connect link", { cause }).pipe( + Effect.as(false), + ), + ), + ) + : false; + if (wantsCliLink) { yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 73612590e5fa..103b6c840a82 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -139,6 +139,27 @@ describe("ManagedEndpointAllocations", () => { }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), ); + it.effect("splits large tunnel lookups into bounded database queries", () => { + const batchSizes: number[] = []; + const fakeDb = { + select: () => ({ + from: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`); + expect(yield* allocations.listByTunnelNames(names)).toEqual([]); + expect(batchSizes).toEqual([500, 500, 1]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { const fakeDb = { update: (table: unknown) => { diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index afd77151a34a..ee1bf35e7990 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -29,6 +29,8 @@ export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocati readonly recoveryEnabled: boolean; } +export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -422,31 +424,45 @@ export const make = Effect.gen(function* () { if (tunnelNames.length === 0) { return []; } - return yield* db - .select({ - ...allocationSelection, - recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, - }) - .from(relayManagedEndpointAllocations) - .where(inArray(relayManagedEndpointAllocations.tunnelName, tunnelNames)) - .pipe( - Effect.map((rows) => - rows.map(({ recoveryEnabledAt, ...allocation }) => ({ - ...allocation, - recoveryEnabled: recoveryEnabledAt !== null, - })), - ), - Effect.mapError( - (cause) => - new ManagedEndpointAllocationPersistenceError({ - operation: "list-tunnels", - stage: "database-request", - userId: "*", - environmentId: "*", - cause, - }), + const batches = Array.from( + { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) }, + (_, index) => + tunnelNames.slice( + index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, ), - ); + ); + const results = yield* Effect.forEach( + batches, + (batch) => + db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + }) + .from(relayManagedEndpointAllocations) + .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) + .pipe( + Effect.map((rows) => + rows.map(({ recoveryEnabledAt, ...allocation }) => ({ + ...allocation, + recoveryEnabled: recoveryEnabledAt !== null, + })), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ), + { concurrency: 1 }, + ); + return results.flat(); }, ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 69d208a276ba..cc0ddc8b5edc 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -1087,6 +1087,64 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not change tunnel ingress after another provision takes ownership", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: () => Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "configure-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change DNS after another provision takes ownership", () => { + const dnsCalls: DnsCall[] = []; + const allocations = makeAllocations(); + let lockCount = 0; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + ++lockCount === 1 + ? allocations.withClaimedTunnel(input, effect) + : Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + expect(dnsCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + it.effect("does not mark a superseded tunnel allocation as ready", () => { const allocations = makeAllocations(); const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index b3752a3b994e..507707213d5a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -937,30 +937,66 @@ export const make = Effect.gen(function* () { }); } - yield* tunnels - .putConfiguration(tunnel.id, { - ingress: [ - { - hostname, - service: formatOriginService(input.origin), - }, - { service: "http_status:404" }, - ], - }) + const configured = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + tunnels + .putConfiguration(tunnel.id, { + ingress: [ + { + hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ), + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "configure-tunnel", - hostname, - tunnelName, - tunnelId: tunnel.id, - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(configured)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed before its configuration was updated.", + }); + } const dnsRecord = { type: "CNAME", @@ -970,33 +1006,61 @@ export const make = Effect.gen(function* () { proxied: true, } as const; - const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "ensure-dns-record", + const recordedDns = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + Effect.gen(function* () { + const dnsRecordId = yield* ensureDnsRecord( hostname, - tunnelName, - tunnelId: tunnel.id, - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), - ); - const dnsGeneration = yield* allocations - .recordDns({ - userId: input.userId, - environmentId: input.environmentId, - dnsRecordId, - tunnelId: tunnel.id, - generation: tunnelGeneration, - }) - .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ + allocation.dnsRecordId, + dnsRecord, + ).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "ensure-dns-record", + hostname, + tunnelName, + tunnelId: tunnel.id, + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + const dnsGeneration = yield* allocations + .recordDns({ + userId: input.userId, + environmentId: input.environmentId, + dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause, + }), + ), + ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, stage: "record-dns", @@ -1004,11 +1068,29 @@ export const make = Effect.gen(function* () { tunnelName, tunnelId: tunnel.id, dnsRecordId, - cause, - }), - ), + cause: "The tunnel allocation changed before its DNS record was saved.", + }); + } + return { dnsRecordId, dnsGeneration }; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); - if (dnsGeneration === null) { + if (Option.isNone(recordedDns)) { return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, @@ -1016,10 +1098,10 @@ export const make = Effect.gen(function* () { hostname, tunnelName, tunnelId: tunnel.id, - dnsRecordId, - cause: "The tunnel allocation changed before its DNS record was saved.", + cause: "The tunnel allocation changed before its DNS record was updated.", }); } + const { dnsRecordId, dnsGeneration } = recordedDns.value; const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( From 8555b7c17210129fac4c97225c042be44393c118 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:44:56 -0700 Subject: [PATCH 06/10] fix(connect): bind tunnel recovery to the current environment --- apps/server/src/cloud/http.test.ts | 30 +++++ apps/server/src/cloud/http.ts | 28 ++++- apps/server/src/server.ts | 2 + .../migration.sql | 1 + .../snapshot.json | 15 ++- .../ManagedEndpointAllocations.test.ts | 107 ++++++++++++++---- .../ManagedEndpointAllocations.ts | 34 +++++- .../ManagedEndpointProvider.test.ts | 1 + infra/relay/src/persistence/schema.ts | 1 + 9 files changed, 187 insertions(+), 32 deletions(-) rename infra/relay/migrations/postgres/{20260825034308_managed_endpoint_recovery => 20260825044249_managed_endpoint_recovery}/migration.sql (63%) rename infra/relay/migrations/postgres/{20260825034308_managed_endpoint_recovery => 20260825044249_managed_endpoint_recovery}/snapshot.json (98%) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index fd0328b2c414..418230da2d35 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -662,6 +662,36 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect.each([ + { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 403, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 409, errorTag: "EnvironmentHttpConflictError" }, + ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773")); + + expect(error._tag).toBe(errorTag); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({}, { status }), + }), + ); + }); + it.effect("keeps a tunnel configuration replaced during recovery", () => { const { store, values } = makeMemorySecretStore([ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 14d913cd105c..fdd59c6d05f0 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -537,13 +537,29 @@ const relayClientRequest = ( HttpClientRequest.bearerToken(input.token), HttpClientRequest.bodyJson(input.payload), Effect.flatMap(dependencies.httpClient.execute), - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status === 401 || response.status === 403) { + return yield* new EnvironmentHttpUnauthorizedError({ + message: "T3 Connect rejected the stored environment credential.", + }); + } + if (response.status === 409) { + return yield* new EnvironmentHttpConflictError({ + message: "T3 Connect rejected the current environment link.", + }); + } + return yield* HttpClientResponse.filterStatusOk(response); + }), + ), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), - Effect.mapError( - (cause) => - new EnvironmentHttpInternalServerError({ - message: `T3 Connect relay request failed: ${String(cause)}`, - }), + Effect.mapError((cause) => + cause._tag === "EnvironmentHttpUnauthorizedError" || + cause._tag === "EnvironmentHttpConflictError" + ? cause + : new EnvironmentHttpInternalServerError({ + message: `T3 Connect relay request failed: ${String(cause)}`, + }), ), withRelayClientTracing, ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0ab72658cff3..c630dcc375b3 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -624,6 +624,8 @@ export const makeServerLayer = Layer.unwrap( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError" && (error._tag !== "EnvironmentCloudEndpointUnavailableError" || CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( error.endpointRuntimeStatus, diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql similarity index 63% rename from infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql rename to infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql index 4da6b524efa4..a9480ed4c93b 100644 --- a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/migration.sql +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql @@ -1,2 +1,3 @@ ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json similarity index 98% rename from infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json rename to infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json index 21afe569ac4e..ce549ba1c6c0 100644 --- a/infra/relay/migrations/postgres/20260825034308_managed_endpoint_recovery/snapshot.json +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json @@ -1,7 +1,7 @@ { "version": "8", "dialect": "postgres", - "id": "a0128e5a-4bba-4f2d-9851-82c3744dae2c", + "id": "7e85c554-d61a-4253-bd7b-17f92e98e665", "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], "ddl": [ { @@ -877,6 +877,19 @@ "schema": "public", "table": "relay_managed_endpoint_allocations" }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, { "type": "integer", "typeSchema": null, diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 103b6c840a82..de14b4b82a97 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -12,10 +12,49 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("clears endpoint readiness when recording a replacement tunnel", () => { + let updated: + | { + readonly tunnelId: string; + readonly readyAt: string | null; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { readonly tunnelId: string; readonly readyAt: string | null }) => { + updated = values; + return { + where: () => ({ + returning: () => Effect.succeed([{ generation: 8 }]), + }), + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.recordTunnel({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + generation: 7, + }), + ).toBe(8); + expect(updated?.tunnelId).toBe("replacement-tunnel"); + expect(updated?.readyAt).toBeNull(); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("records recovery support and advances the allocation generation", () => { let updated: | { readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; readonly updatedAt: string; } | undefined; @@ -24,7 +63,11 @@ describe("ManagedEndpointAllocations", () => { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly recoveryEnabledAt: string; readonly updatedAt: string }) => { + set: (values: { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + }) => { updated = values; return { where: (where: unknown) => { @@ -52,6 +95,7 @@ describe("ManagedEndpointAllocations", () => { expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); expect(updated?.recoveryEnabledAt).toBeDefined(); + expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key"); const query = new PgDialect().sqlToQuery(condition as never); expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); @@ -101,21 +145,35 @@ describe("ManagedEndpointAllocations", () => { from: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - where: () => - Effect.succeed([ - { - ...base, - environmentId: "environment-1", - tunnelId: "tunnel-1", - recoveryEnabledAt: "2026-08-25T12:00:00.000Z", - }, - { - ...base, - environmentId: "environment-2", - tunnelId: "tunnel-2", - recoveryEnabledAt: null, - }, - ]), + leftJoin: () => ({ + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + recoveryEnvironmentPublicKey: null, + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-3", + tunnelId: "tunnel-3", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "old-key", + linkedEnvironmentPublicKey: "new-key", + }, + ]), + }), }; }, }), @@ -123,11 +181,16 @@ describe("ManagedEndpointAllocations", () => { return Effect.gen(function* () { const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; - const result = yield* allocations.listByTunnelNames(["first-tunnel", "second-tunnel"]); + const result = yield* allocations.listByTunnelNames([ + "first-tunnel", + "second-tunnel", + "third-tunnel", + ]); expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ ["tunnel-1", true], ["tunnel-2", false], + ["tunnel-3", false], ]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); @@ -144,10 +207,12 @@ describe("ManagedEndpointAllocations", () => { const fakeDb = { select: () => ({ from: () => ({ - where: (condition: unknown) => { - batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); - return Effect.succeed([]); - }, + leftJoin: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), }), }), } as unknown as RelayDb.RelayDb["Service"]; diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index ee1bf35e7990..fda5aae35a50 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -288,6 +288,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, + readyAt: null, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -382,6 +383,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ recoveryEnabledAt: now, + recoveryEnvironmentPublicKey: input.environmentPublicKey, updatedAt: now, generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) @@ -439,15 +441,39 @@ export const make = Effect.gen(function* () { .select({ ...allocationSelection, recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + recoveryEnvironmentPublicKey: + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, }) .from(relayManagedEndpointAllocations) + .leftJoin( + relayEnvironmentLinks, + and( + eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId), + eq( + relayEnvironmentLinks.environmentId, + relayManagedEndpointAllocations.environmentId, + ), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) .pipe( Effect.map((rows) => - rows.map(({ recoveryEnabledAt, ...allocation }) => ({ - ...allocation, - recoveryEnabled: recoveryEnabledAt !== null, - })), + rows.map( + ({ + recoveryEnabledAt, + recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey, + ...allocation + }) => ({ + ...allocation, + recoveryEnabled: + recoveryEnabledAt !== null && + recoveryEnvironmentPublicKey !== null && + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, + }), + ), ), Effect.mapError( (cause) => diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index cc0ddc8b5edc..8131b20a635b 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -234,6 +234,7 @@ function makeAllocations(calls: AllocationCall[] = []) { mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, + readyAt: null, })); return allocations.get(allocationKey(input))?.generation ?? null; }), diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index a53be1ea1a30..88196edc4fd6 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -94,6 +94,7 @@ export const relayManagedEndpointAllocations = pgTable( dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + recoveryEnvironmentPublicKey: text("recovery_environment_public_key"), generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), From f707f1b593bd56c64d26ccdd0a6af57742106a33 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 24 Aug 2026 21:53:33 -0700 Subject: [PATCH 07/10] fix(connect): keep existing tunnels ready during recovery --- .../environments/ManagedEndpointAllocations.test.ts | 12 ++++++++---- .../src/environments/ManagedEndpointAllocations.ts | 2 +- .../src/environments/ManagedEndpointProvider.test.ts | 9 +++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index de14b4b82a97..843604e5c4eb 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -12,18 +12,18 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { - it.effect("clears endpoint readiness when recording a replacement tunnel", () => { + it.effect("clears endpoint readiness only when the recorded tunnel changes", () => { let updated: | { readonly tunnelId: string; - readonly readyAt: string | null; + readonly readyAt: unknown; } | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly tunnelId: string; readonly readyAt: string | null }) => { + set: (values: { readonly tunnelId: string; readonly readyAt: unknown }) => { updated = values; return { where: () => ({ @@ -46,7 +46,11 @@ describe("ManagedEndpointAllocations", () => { }), ).toBe(8); expect(updated?.tunnelId).toBe("replacement-tunnel"); - expect(updated?.readyAt).toBeNull(); + const query = new PgDialect().sqlToQuery(updated?.readyAt as never); + expect(query.sql).toBe( + 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end', + ); + expect(query.params).toEqual(["replacement-tunnel"]); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index fda5aae35a50..bd764071c308 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -288,7 +288,7 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, - readyAt: null, + readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`, updatedAt: DateTime.formatIso(yield* DateTime.now), generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 8131b20a635b..c7ae9c0d1569 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -234,7 +234,7 @@ function makeAllocations(calls: AllocationCall[] = []) { mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, - readyAt: null, + readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null, })); return allocations.get(allocationKey(input))?.generation ?? null; }), @@ -807,7 +807,8 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.andThen(Effect.fail(failure))), deleteRecord: () => Effect.void, }); - const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations()); + const allocations = makeAllocations(); + const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -830,6 +831,10 @@ describe("ManagedEndpointProvider", () => { "createRecord", "updateRecord", ]); + expect(yield* allocations.get(request)).toMatchObject({ + tunnelId: "tunnel-id", + readyAt: "2026-06-02T00:00:00.000Z", + }); }).pipe(Effect.provide(layer)); }); From a60f0d9d3fd6ed4f13342f2f67feef80d5a92ad8 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:10:56 -0700 Subject: [PATCH 08/10] fix(connect): recover tunnels without startup provisioning --- .../src/cloud/ManagedEndpointRuntime.test.ts | 114 +++++++++++++++- .../src/cloud/ManagedEndpointRuntime.ts | 47 ++++++- apps/server/src/cloud/http.test.ts | 65 ++++++++- apps/server/src/cloud/http.ts | 38 ++++++ apps/server/src/server.ts | 31 ++++- docs/internals/t3-connect.md | 27 ++-- docs/user/remote-access.md | 3 +- infra/relay/src/http/Api.test.ts | 123 ++++++++++++++++++ infra/relay/src/http/Api.ts | 103 ++++++++++++--- packages/contracts/src/relay.ts | 19 +++ 10 files changed, 525 insertions(+), 45 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 40f8a271dd5a..733c19e11442 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -6,9 +6,11 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -60,6 +62,7 @@ function makeHandle(input: { readonly onKill: () => void; readonly isRunning?: () => boolean; readonly exitCode?: Effect.Effect; + readonly output?: Stream.Stream; }) { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(input.pid), @@ -73,7 +76,7 @@ function makeHandle(input: { stdin: Sink.drain, stdout: Stream.empty, stderr: Stream.empty, - all: Stream.empty, + all: input.output ?? Stream.empty, getInputFd: () => Sink.drain, getOutputFd: () => Stream.empty, }); @@ -164,6 +167,115 @@ describe("CloudManagedEndpointRuntime", () => { ).toBe("warning"); }); + it("recognizes tunnel authorization failures without matching ordinary transport errors", () => { + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', + ), + ).toBe(false); + }); + + it.effect("keeps recovery requests sent before the server starts consuming them", () => + Effect.gen(function* () { + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "tunnel-1", + }; + + yield* runtime.requestRecovery(config); + + expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config); + }), + ); + + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const firstBatchObserved = yield* Deferred.make(); + const secondBatchObserved = yield* Deferred.make(); + const recoveryRequested = yield* Deferred.make(); + const spawned: Array = []; + const encoder = new TextEncoder(); + const connectorOutput = Stream.fromQueue(output).pipe( + Stream.tap((chunk) => { + const line = new TextDecoder().decode(chunk); + if (line === "first checkpoint\n") { + return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid); + } + if (line === "second checkpoint\n") { + return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid); + } + return Effect.void; + }), + ); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const pid = 600; + spawned.push(pid); + const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "deleted-tunnel", + }; + const rejectedLine = + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0\n'; + + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => + Deferred.succeed(recoveryRequested, requested).pipe(Effect.asVoid), + ), + Effect.forkChild, + ); + yield* runtime.applyConfig(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3))); + yield* Queue.offer(output, encoder.encode("first checkpoint\n")); + yield* Deferred.await(firstBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer( + output, + encoder.encode( + "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" + + rejectedLine.repeat(3), + ), + ); + yield* Queue.offer(output, encoder.encode("second checkpoint\n")); + yield* Deferred.await(secondBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer(output, encoder.encode(rejectedLine)); + + expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + expect(spawned).toEqual([600]); + }), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index ca12b430649f..3079428bb117 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -5,7 +5,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -73,6 +73,9 @@ interface ActiveConnector { readonly config: RelayManagedEndpointRuntimeConfig; } +// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. +const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; + export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" { if (/\bRegistered tunnel connection\b/iu.test(line)) { return "connected"; @@ -83,6 +86,15 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +export function isRejectedRelayClientTunnelOutput(line: string): boolean { + return ( + /\bRegister tunnel error from server side\b/iu.test(line) && + /\bUnauthorized:\s*(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)\b/iu.test( + line, + ) + ); +} + /** Connector startup failures can clear after installation or a later spawn attempt. */ export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { if (typeof status !== "object" || status === null || !("status" in status)) { @@ -120,7 +132,7 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); - const recoveryRequests = yield* PubSub.sliding(1); + const recoveryRequests = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; @@ -162,7 +174,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); - yield* PubSub.publish(recoveryRequests, connector.config); + yield* Queue.offer(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -170,8 +182,11 @@ export const make = Effect.gen(function* () { Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), ); - const observeConnectorOutput = (connector: ActiveConnector) => - connector.child.all.pipe( + const observeConnectorOutput = (connector: ActiveConnector) => { + let rejectedRegistrations = 0; + let recoveryRequested = false; + + return connector.child.all.pipe( Stream.decodeText(), Stream.splitLines, Stream.map((line) => line.trim()), @@ -186,8 +201,25 @@ export const make = Effect.gen(function* () { }; switch (classifyRelayClientOutput(line)) { case "connected": + rejectedRegistrations = 0; return Effect.logInfo("Relay client tunnel connection registered", attributes); case "warning": + if (isRejectedRelayClientTunnelOutput(line)) { + rejectedRegistrations += 1; + if ( + !recoveryRequested && + rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY + ) { + recoveryRequested = true; + return Effect.logWarning( + "Relay client tunnel was rejected; requesting recovery", + attributes, + ).pipe( + Effect.andThen(Queue.offer(recoveryRequests, connector.config)), + Effect.asVoid, + ); + } + } return Effect.logWarning("Relay client reported a transport warning", attributes); case "debug": return Effect.logDebug("Relay client output", attributes); @@ -202,6 +234,7 @@ export const make = Effect.gen(function* () { }), ), ); + }; reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) { if (!config || config.providerKind !== "cloudflare_tunnel") { @@ -327,8 +360,8 @@ export const make = Effect.gen(function* () { const runtime = CloudManagedEndpointRuntime.of({ applyConfig, - recoveryRequests: Stream.fromPubSub(recoveryRequests), - requestRecovery: (config) => PubSub.publish(recoveryRequests, config).pipe(Effect.asVoid), + recoveryRequests: Stream.fromQueue(recoveryRequests), + requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), withLinkStateLock: linkStateSemaphore.withPermits(1), }); diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 418230da2d35..c27c6a64a4f8 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -6,6 +6,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; import * as Stream from "effect/Stream"; import { @@ -30,7 +31,10 @@ import { import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; -import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; +import { + RelayManagedEndpointRecoveryRegistrationRequest, + type RelayLinkProofRequest, +} from "@t3tools/contracts/relay"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, @@ -45,6 +49,7 @@ import { pendingServiceUpdateExists, reconcileDesiredCloudLink, recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -62,6 +67,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") => }); const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation"); +const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( + Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), +); function makeSecretStore( create: ServerSecretStore.ServerSecretStore["Service"]["create"], @@ -600,6 +608,61 @@ describe("releaseManagedTunnelOnShutdown", () => { ); }); + it.effect("registers an existing tunnel without provisioning or restarting it", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe( + "https://relay.example.test/v1/environments/env_123/tunnel/recovery", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + const body = requests[0]?.body; + expect(body?._tag).toBe("Uint8Array"); + if (body?._tag === "Uint8Array") { + expect( + yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), + ).toEqual({ + cloudUserId: "user-123", + tunnelId: "existing-tunnel", + }); + } + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("does not register recovery without a recorded tunnel ID", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("recovers a web-linked tunnel with its environment credential", () => { const oldConfig = '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index fdd59c6d05f0..b99c77368d18 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -71,6 +71,7 @@ import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, + decodeRuntimeConfig, encodeEndpointRuntimeConfigJson, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, @@ -669,6 +670,43 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +export const registerManagedCloudTunnelRecovery = Effect.fn( + "environment.cloud.registerManagedCloudTunnelRecovery", +)(function* () { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if (config?.providerKind !== "cloudflare_tunnel" || config.tunnelId === undefined) { + return false; + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const registered = yield* relayClientRequest(dependencies, { + url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: bytesToString(cloudUserId.value), + tunnelId: config.tunnelId, + }, + schema: RelayOkResponse, + }); + return registered.ok; +}); + export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( function* (localOrigin: string) { const dependencies = yield* cloudHttpDependencies; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c630dcc375b3..9467cd0f9986 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -96,6 +96,7 @@ import { pendingServiceUpdateExists, reconcileDesiredCloudLink, recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; @@ -668,8 +669,27 @@ export const makeServerLayer = Layer.unwrap( ), ) : false; - if (wantsCliLink) { - yield* reconcileDesiredCloudLink(localOrigin).pipe( + const desiredCliLinkMode = wantsCliLink + ? yield* CloudCliState.readCliDesiredLinkMode + : null; + const registerManagedTunnel = registerManagedCloudTunnelRecovery().pipe( + Effect.tap((registered) => + registered + ? Effect.logInfo("T3 Connect managed tunnel recovery registered") + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as(false)), + ), + ); + const registered = + desiredCliLinkMode === "publish_only" ? false : yield* registerManagedTunnel; + if (wantsCliLink && !registered) { + const reconciled = yield* reconcileDesiredCloudLink(localOrigin).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && @@ -683,14 +703,17 @@ export const makeServerLayer = Layer.unwrap( ), }), Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.as(true), Effect.catch((cause) => Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { cause, - }), + }).pipe(Effect.as(false)), ), ); + if (reconciled && desiredCliLinkMode === "managed") { + yield* registerManagedTunnel; + } } - yield* recoverManagedTunnel; }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index e342f5adba62..16edf0c65b55 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -127,17 +127,22 @@ logout` performs the same cleanup and removes the stored CLI authorization. ### Managed tunnel lifecycle -Every linked environment stores a relay-issued environment credential. When a managed environment -starts or its connector exits, the server uses that credential to request a tunnel from the relay. -This also covers environments linked through web or mobile settings, which do not have a stored CLI -credential. The relay keeps the existing hostname and DNS record, so a replacement tunnel does not -change the public endpoint. - -After a host completes this recovery request, the relay records that the host can recreate its own -tunnel. The existing five-minute maintenance job removes tunnels from those hosts when Cloudflare -reports that they have been down for at least five minutes. Tunnels that never connected are removed -when they are at least five minutes old. The job leaves older hosts alone until they complete a -recovery request, and it only removes tunnels that belong to its own deployment stage. +Every linked environment stores a relay-issued environment credential. At startup, the server uses +that credential to register recovery support for its existing tunnel. Registration only updates the +relay database and does not call Cloudflare. Healthy CLI links reuse the stored tunnel instead of +provisioning it again. + +If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses +the same environment credential to request a replacement. This also recovers a tunnel deleted +while a laptop was asleep, even when the connector keeps running. Environments linked through web +or mobile settings do not need a stored CLI credential. The relay keeps the existing hostname and +DNS record, so a replacement tunnel does not change the public endpoint. + +After a host registers recovery support, the existing five-minute maintenance job removes its +tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels +that never connected are removed when they are at least five minutes old. The job leaves older +hosts alone until they register recovery support, and it only removes tunnels that belong to its +own deployment stage. The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index ff6fc4285fb5..67259f8cce26 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -231,7 +231,8 @@ controls remain in **Settings** → **Connections** on web and desktop or **Sett If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. The environment stays linked to your account and keeps the same address. When the server starts -again, T3 Connect creates a replacement tunnel automatically. You do not need to pair it again. +again or the computer wakes, T3 Connect creates a replacement tunnel automatically. You do not +need to pair it again. ## Security Notes diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index f759c9166e2e..883c366a511a 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -24,6 +24,7 @@ import { relayEnvironmentAuthLayer, relayNotFoundRoute, recoverEnvironmentTunnelRecord, + registerEnvironmentTunnelRecovery, revokeEnvironmentLinkRecord, traceRelayHttpRequestWith, unlinkEnvironmentRecord, @@ -224,6 +225,107 @@ const linkedEnvironmentRecord = { } as const; describe("relay managed tunnel recovery", () => { + it.effect("registers recovery for an existing tunnel without provisioning it", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "existing-tunnel", + }), + ).toEqual({ ok: true }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "existing-tunnel", + environmentPublicKey: "public-key", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => Effect.die("registration must not provision a tunnel"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration for a different environment key", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + tunnelId: "existing-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration when the recorded tunnel changed", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "stale-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ), + ); + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { let recoveryEnabledFor: { readonly userId: string; @@ -348,6 +450,18 @@ describe("relay managed tunnel recovery", () => { it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { let recoveryEnabled = false; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "different.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "replacement-generation", + generation: 3, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { const error = yield* Effect.flip( @@ -360,6 +474,7 @@ describe("relay managed tunnel recovery", () => { ); expect(error).toMatchObject({ _tag: "Unauthorized" }); expect(recoveryEnabled).toBe(false); + expect(cleaned).toEqual(["replacement-tunnel"]); }).pipe( Effect.provide( Layer.merge( @@ -378,6 +493,14 @@ describe("relay managed tunnel recovery", () => { tunnelId: "replacement-tunnel", }, }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; + }), }), Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ enableRecovery: () => diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 58aefda36b60..d537b15a5983 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -471,6 +471,33 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +export const registerEnvironmentTunnelRecovery = Effect.fn( + "relay.api.server.registerEnvironmentTunnelRecovery", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly tunnelId: string; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (!(yield* allocations.enableRecovery(input))) { + return yield* new HttpApiError.Unauthorized({}); + } + return { ok: true }; +}); + export const recoverEnvironmentTunnelRecord = Effect.fn( "relay.api.server.recoverEnvironmentTunnelRecord", )(function* (input: { @@ -505,6 +532,22 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { + if (recoveredTunnelId !== undefined) { + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); + if (target?.tunnelId === recoveredTunnelId) { + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + cause, + }), + ), + ); + } + } return yield* new HttpApiError.Unauthorized({}); } @@ -1057,35 +1100,55 @@ export const serverApi = HttpApiBuilder.group( ), ); - return activityHandlers.handle( - "recoverManagedEndpoint", - Effect.fn("relay.api.server.recoverManagedEndpoint")( - function* ({ params, payload }) { + return activityHandlers + .handle( + "registerManagedEndpointRecovery", + Effect.fn("relay.api.server.registerManagedEndpointRecovery")(function* ({ + params, + payload, + }) { const principal = yield* RelayEnvironmentPrincipal; if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } yield* appendRelayCredentialResponseHeaders; - return yield* recoverEnvironmentTunnelRecord({ + return yield* registerEnvironmentTunnelRecovery({ userId: payload.cloudUserId, environmentId: params.environmentId, environmentPublicKey: principal.environmentPublicKey, - origin: payload.origin, + tunnelId: payload.tunnelId, }); - }, - Effect.catchTags({ - ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), - ManagedEndpointProvisioningNotConfigured: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedEndpointProvisioningFailed: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedEndpointDeprovisioningFailed: () => - relayInternalErrorResponse("upstream_unavailable"), - ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), - }), - mapRelayCommonApiErrors("not_authorized"), - ), - ); + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 76b0b0ef7438..cd1ea10228e1 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -165,6 +165,13 @@ export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ }); export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; +export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRegistrationRequest = + typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; + export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ endpoint: RelayManagedEndpoint, endpointRuntime: RelayManagedEndpointRuntimeConfig, @@ -1065,6 +1072,18 @@ export const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") export const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post( + "registerManagedEndpointRecovery", + "/v1/environments/:environmentId/tunnel/recovery", + { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRegistrationRequest, + success: RelayOkResponse, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"), HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { params: Schema.Struct({ environmentId: EnvironmentId, From a9386bf2034a8b363fbb258547fb1ef58e991acf Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:34:53 -0700 Subject: [PATCH 09/10] fix(connect): sign tunnel recovery requests --- .../src/cloud/ManagedEndpointRuntime.test.ts | 16 ++- .../src/cloud/ManagedEndpointRuntime.ts | 8 +- apps/server/src/cloud/http.test.ts | 33 +++++- apps/server/src/cloud/http.ts | 110 ++++++++++++++++-- apps/server/src/server.ts | 58 ++++----- docs/internals/t3-connect.md | 12 +- infra/relay/src/http/Api.test.ts | 89 +++++++++++--- infra/relay/src/http/Api.ts | 89 +++++++++++++- packages/contracts/src/relay.ts | 21 ++++ packages/shared/src/relayJwt.ts | 1 + 10 files changed, 361 insertions(+), 76 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 733c19e11442..cb2441095f5d 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -213,6 +213,8 @@ describe("CloudManagedEndpointRuntime", () => { const firstBatchObserved = yield* Deferred.make(); const secondBatchObserved = yield* Deferred.make(); const recoveryRequested = yield* Deferred.make(); + const recoveryRetried = yield* Deferred.make(); + let recoveryRequestCount = 0; const spawned: Array = []; const encoder = new TextEncoder(); const connectorOutput = Stream.fromQueue(output).pipe( @@ -246,9 +248,13 @@ describe("CloudManagedEndpointRuntime", () => { '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0\n'; yield* runtime.recoveryRequests.pipe( - Stream.runForEach((requested) => - Deferred.succeed(recoveryRequested, requested).pipe(Effect.asVoid), - ), + Stream.runForEach((requested) => { + recoveryRequestCount += 1; + return Deferred.succeed( + recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried, + requested, + ).pipe(Effect.asVoid); + }), Effect.forkChild, ); yield* runtime.applyConfig(config); @@ -272,6 +278,10 @@ describe("CloudManagedEndpointRuntime", () => { yield* Queue.offer(output, encoder.encode(rejectedLine)); expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4))); + + expect(yield* Deferred.await(recoveryRetried)).toEqual(config); expect(spawned).toEqual([600]); }), ); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 3079428bb117..7695327da63b 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -184,7 +184,6 @@ export const make = Effect.gen(function* () { const observeConnectorOutput = (connector: ActiveConnector) => { let rejectedRegistrations = 0; - let recoveryRequested = false; return connector.child.all.pipe( Stream.decodeText(), @@ -206,11 +205,8 @@ export const make = Effect.gen(function* () { case "warning": if (isRejectedRelayClientTunnelOutput(line)) { rejectedRegistrations += 1; - if ( - !recoveryRequested && - rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY - ) { - recoveryRequested = true; + if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) { + rejectedRegistrations = 0; return Effect.logWarning( "Relay client tunnel was rejected; requesting recovery", attributes, diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index c27c6a64a4f8..6e500aec5816 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -270,7 +270,10 @@ describe("releaseManagedTunnelOnShutdown", () => { Effect.sync(() => { values.set(name, value); }), - create: unusedSecretStoreOperation, + create: (name, value) => + Effect.sync(() => { + values.set(name, value); + }), getOrCreateRandom: unusedSecretStoreOperation, remove: (name) => Effect.sync(() => { @@ -634,7 +637,7 @@ describe("releaseManagedTunnelOnShutdown", () => { if (body?._tag === "Uint8Array") { expect( yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), - ).toEqual({ + ).toMatchObject({ cloudUserId: "user-123", tunnelId: "existing-tunnel", }); @@ -725,6 +728,32 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect( + yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", { + providerKind: "cloudflare_tunnel", + connectorToken: "old-token", + tunnelId: "old-tunnel", + }), + ).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect.each([ { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, { status: 403, errorTag: "EnvironmentHttpUnauthorizedError" }, diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b99c77368d18..a0cbcafa5a4a 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,7 +28,9 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, RelayManagedEndpointRecoveryResponse, + type RelayManagedEndpointRuntimeConfig, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -37,6 +39,7 @@ import { RELAY_HEALTH_REQUEST_TYP, RELAY_HEALTH_RESPONSE_TYP, RELAY_LINK_PROOF_TYP, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, RELAY_MINT_REQUEST_TYP, RELAY_MINT_RESPONSE_TYP, signRelayJwt, @@ -496,7 +499,14 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); - if (options?.requestRecovery !== false) { + const registered = yield* registerManagedCloudTunnelRecovery().pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as(false)), + ), + ); + if (!registered && options?.requestRecovery !== false) { yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); } } else { @@ -670,6 +680,53 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD }, ); +type ManagedTunnelRecoveryProofInput = { + readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"]; + readonly cloudUserId: string; + readonly relayUrl: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +const makeManagedTunnelRecoveryProof = Effect.fn( + "environment.cloud.makeManagedTunnelRecoveryProof", +)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) { + const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets); + const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const claims = { + iss: `t3-env:${input.environmentId}`, + aud: normalizeRelayIssuer( + Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl, + ), + sub: input.environmentId, + jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)), + iat: issuedAt, + exp: issuedAt + 60, + environmentId: input.environmentId, + cloudUserId: input.cloudUserId, + }; + const payload = + input.action === "register" + ? { ...claims, action: "register" as const, tunnelId: input.tunnelId } + : { ...claims, action: "recover" as const, origin: input.origin }; + + return yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload, + }).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not sign the managed tunnel recovery request.", + }), + ), + ); +}); + export const registerManagedCloudTunnelRecovery = Effect.fn( "environment.cloud.registerManagedCloudTunnelRecovery", )(function* () { @@ -695,12 +752,22 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( } const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "register", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + tunnelId: config.tunnelId, + }); const registered = yield* relayClientRequest(dependencies, { - url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, token: bytesToString(environmentCredential.value), payload: { - cloudUserId: bytesToString(cloudUserId.value), + cloudUserId: cloudUserIdValue, tunnelId: config.tunnelId, + proof, }, schema: RelayOkResponse, }); @@ -708,7 +775,7 @@ export const registerManagedCloudTunnelRecovery = Effect.fn( }); export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( - function* (localOrigin: string) { + function* (localOrigin: string, expectedConfig?: RelayManagedEndpointRuntimeConfig) { const dependencies = yield* cloudHttpDependencies; const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), @@ -724,6 +791,18 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan ) { return false; } + if (expectedConfig !== undefined) { + const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if ( + current === null || + current.providerKind !== expectedConfig.providerKind || + current.connectorToken !== expectedConfig.connectorToken || + current.tunnelId !== expectedConfig.tunnelId || + current.tunnelName !== expectedConfig.tunnelName + ) { + return false; + } + } const localUrl = yield* Effect.try({ try: () => new URL(localOrigin), @@ -739,15 +818,26 @@ export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverMan } const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const origin = { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }; + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "recover", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + origin, + }); const recovered = yield* relayClientRequest(dependencies, { - url: `${bytesToString(relayUrl.value)}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, token: bytesToString(environmentCredential.value), payload: { - cloudUserId: bytesToString(cloudUserId.value), - origin: { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }, + cloudUserId: cloudUserIdValue, + origin, + proof, }, schema: RelayManagedEndpointRecoveryResponse, }); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 9467cd0f9986..1b2dc519867f 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,4 +1,5 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; @@ -620,38 +621,39 @@ export const makeServerLayer = Layer.unwrap( const localOrigin = `http://127.0.0.1:${address.port}`; const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; const recoveryLock = yield* Semaphore.make(1); - const recoverManagedTunnel = recoveryLock.withPermits(1)( - recoverManagedCloudTunnel(localOrigin).pipe( - Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError" && - (error._tag !== "EnvironmentCloudEndpointUnavailableError" || - CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( - error.endpointRuntimeStatus, - )), - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => + recoveryLock.withPermits(1)( + recoverManagedCloudTunnel(localOrigin, config).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError" && + (error._tag !== "EnvironmentCloudEndpointUnavailableError" || + CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + )), + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), ), - Schedule.upTo({ duration: "10 minutes" }), + }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { + cause, + }), ), - }), - Effect.tap((recovered) => - recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, - ), - Effect.catchCause((cause) => - Cause.hasInterrupts(cause) - ? Effect.interrupt - : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { - cause, - }), ), - ), - ); + ); yield* endpointRuntime.recoveryRequests.pipe( - Stream.runForEach(() => recoverManagedTunnel), + Stream.runForEach(recoverManagedTunnel), Effect.forkScoped, ); // No settling delay before the first attempt: routes are already diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 16edf0c65b55..0c7ed2ba3096 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -127,16 +127,18 @@ logout` performs the same cleanup and removes the stored CLI authorization. ### Managed tunnel lifecycle -Every linked environment stores a relay-issued environment credential. At startup, the server uses -that credential to register recovery support for its existing tunnel. Registration only updates the -relay database and does not call Cloudflare. Healthy CLI links reuse the stored tunnel instead of -provisioning it again. +Every linked environment stores a relay-issued environment credential. When setup installs a tunnel +and when the server starts, the server uses that credential to register recovery support for the +existing tunnel. Registration only updates the relay database and does not call Cloudflare. +Healthy CLI links reuse the stored tunnel instead of provisioning it again. If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses the same environment credential to request a replacement. This also recovers a tunnel deleted while a laptop was asleep, even when the connector keeps running. Environments linked through web or mobile settings do not need a stored CLI credential. The relay keeps the existing hostname and -DNS record, so a replacement tunnel does not change the public endpoint. +DNS record, so a replacement tunnel does not change the public endpoint. Each registration and +recovery request includes a short-lived host signature that binds the cloud user and the current +tunnel or local T3 server address. After a host registers recovery support, the existing five-minute maintenance job removes its tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 883c366a511a..4f1c15ff6bcf 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1,7 +1,9 @@ +import * as NodeCrypto from "node:crypto"; import { createClerkClient, verifyToken } from "@clerk/backend"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -16,6 +18,7 @@ import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; import { EnvironmentId } from "@t3tools/contracts"; import { RelayEnvironmentAuth } from "@t3tools/contracts/relay"; +import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { RELAY_REQUEST_DEADLINE_MS, @@ -29,6 +32,7 @@ import { traceRelayHttpRequestWith, unlinkEnvironmentRecord, verifyRelayClientBearerToken, + verifyEnvironmentTunnelRecoveryProof, withoutCapturedParentSpan, } from "./Api.ts"; import * as RelayConfiguration from "../Config.ts"; @@ -172,6 +176,7 @@ function relayUnlinkTestLayer(input?: { readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; + readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; }) { return Layer.mergeAll( Layer.succeed( @@ -206,7 +211,7 @@ function relayUnlinkTestLayer(input?: { provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), deprovision: input?.deprovision ?? (() => Effect.succeed(true)), - release: () => Effect.die("unused release"), + release: input?.release ?? (() => Effect.die("unused release")), }), ), ); @@ -225,6 +230,68 @@ const linkedEnvironmentRecord = { } as const; describe("relay managed tunnel recovery", () => { + it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "recovery-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "recover", + environmentId: "environment-1", + cloudUserId: "user-1", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + const request = { + action: "recover" as const, + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }; + + yield* verifyEnvironmentTunnelRecoveryProof(request); + + const wrongOwner = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }), + ); + expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" }); + + const wrongOrigin = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" }); + + const wrongAction = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + }), + ); + expect(wrongAction).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + it.effect("registers recovery for an existing tunnel without provisioning it", () => { let recoveryEnabledFor: { readonly userId: string; @@ -451,17 +518,6 @@ describe("relay managed tunnel recovery", () => { it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { let recoveryEnabled = false; const cleaned: Array = []; - const target = { - userId: "user-1", - environmentId: "environment-1", - hostname: "different.example.test", - tunnelId: "replacement-tunnel", - tunnelName: "environment-1-tunnel", - dnsRecordId: "dns-1", - readyAt: "2026-07-28T00:00:00.000Z", - updatedAt: "replacement-generation", - generation: 3, - } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { const error = yield* Effect.flip( @@ -493,11 +549,12 @@ describe("relay managed tunnel recovery", () => { tunnelId: "replacement-tunnel", }, }), - prepareDeprovision: () => Effect.succeed(target), - deprovision: ({ target: captured }) => + prepareDeprovision: () => Effect.die("must keep the active allocation"), + deprovision: () => Effect.die("must keep the active link DNS"), + release: ({ expectedTunnelId }) => Effect.sync(() => { - if (captured?.tunnelId) { - cleaned.push(captured.tunnelId); + if (expectedTunnelId) { + cleaned.push(expectedTunnelId); } return true; }), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index d537b15a5983..ce568652efd9 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -47,10 +47,15 @@ import { RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, type RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; -import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt"; +import { + normalizeRelayIssuer, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts"; import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts"; @@ -93,6 +98,10 @@ const relayCorsPreflightHeaders = { "access-control-max-age": "86400", } as const; +const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect( + RelayManagedEndpointRecoveryProofPayload, +); + const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler( (_request, response) => Effect.succeed( @@ -471,6 +480,56 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron }, ); +type EnvironmentTunnelRecoveryProofInput = { + readonly proof: string; + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +export const verifyEnvironmentTunnelRecoveryProof = Effect.fn( + "relay.api.server.verifyEnvironmentTunnelRecoveryProof", +)(function* (input: EnvironmentTunnelRecoveryProofInput) { + const config = yield* RelayConfiguration.RelayConfiguration; + const now = yield* DateTime.now; + const verified = yield* verifyRelayJwt({ + publicKey: input.environmentPublicKey, + token: input.proof, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + issuer: `t3-env:${input.environmentId}`, + audience: normalizeRelayIssuer(config.relayIssuer), + nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), + }).pipe( + Effect.flatMap(decodeManagedTunnelRecoveryProof), + Effect.mapError(() => new HttpApiError.Unauthorized({})), + ); + + if ( + verified.environmentId !== input.environmentId || + verified.sub !== input.environmentId || + verified.cloudUserId !== input.userId || + verified.action !== input.action + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (input.action === "register") { + if (verified.action !== "register" || verified.tunnelId !== input.tunnelId) { + return yield* new HttpApiError.Unauthorized({}); + } + return; + } + if ( + verified.action !== "recover" || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { + return yield* new HttpApiError.Unauthorized({}); + } +}); + export const registerEnvironmentTunnelRecovery = Effect.fn( "relay.api.server.registerEnvironmentTunnelRecovery", )(function* (input: { @@ -533,10 +592,13 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl ) { if (recoveredTunnelId !== undefined) { - const owner = { userId: input.userId, environmentId: input.environmentId }; - const target = yield* managedEndpointProvider.prepareDeprovision(owner); - if (target?.tunnelId === recoveredTunnelId) { - yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + yield* managedEndpointProvider + .release({ + userId: input.userId, + environmentId: input.environmentId, + expectedTunnelId: recoveredTunnelId, + }) + .pipe( Effect.catch((cause) => Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { userId: input.userId, @@ -546,7 +608,6 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( }), ), ); - } } return yield* new HttpApiError.Unauthorized({}); } @@ -1111,6 +1172,14 @@ export const serverApi = HttpApiBuilder.group( if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + }); yield* appendRelayCredentialResponseHeaders; return yield* registerEnvironmentTunnelRecovery({ userId: payload.cloudUserId, @@ -1128,6 +1197,14 @@ export const serverApi = HttpApiBuilder.group( if (principal.environmentId !== params.environmentId) { return yield* new HttpApiError.Unauthorized({}); } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "recover", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); yield* appendRelayCredentialResponseHeaders; return yield* recoverEnvironmentTunnelRecord({ userId: payload.cloudUserId, diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index cd1ea10228e1..8db9dd50aac0 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -162,12 +162,14 @@ export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRunti export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ cloudUserId: TrimmedNonEmptyString, origin: RelayManagedEndpointOrigin, + proof: TrimmedNonEmptyString, }); export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ cloudUserId: TrimmedNonEmptyString, tunnelId: TrimmedNonEmptyString, + proof: TrimmedNonEmptyString, }); export type RelayManagedEndpointRecoveryRegistrationRequest = typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; @@ -205,6 +207,25 @@ const RelaySignedJwtRegisteredClaims = { exp: Schema.Int, } as const; +export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([ + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("register"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("recover"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + }), +]); +export type RelayManagedEndpointRecoveryProofPayload = + typeof RelayManagedEndpointRecoveryProofPayload.Type; + export const RelayAgentActivityPublishProofPayload = Schema.Struct({ ...RelaySignedJwtRegisteredClaims, environmentId: EnvironmentId, diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index 9e848bedfb02..986bd982e622 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt"; export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; +export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; export class RelayJwtError extends Schema.TaggedErrorClass()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]), From 4831d73d7187e0e424e9a4138ba88ce17b2c2638 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 25 Aug 2026 00:48:03 -0700 Subject: [PATCH 10/10] fix(connect): keep recovery limited to managed links --- apps/server/src/cloud/http.ts | 17 ++++++++++------- apps/server/src/server.test.ts | 9 +-------- .../ManagedEndpointAllocations.test.ts | 2 ++ .../environments/ManagedEndpointAllocations.ts | 1 + infra/relay/src/http/Api.test.ts | 17 +++++++++++++++-- infra/relay/src/http/Api.ts | 8 ++++++-- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index a0cbcafa5a4a..2adc803bb4db 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -458,7 +458,7 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, - options?: { readonly requestRecovery?: boolean; readonly lockHeld?: boolean }, + options?: { readonly lockHeld?: boolean }, ) { const apply = Effect.gen(function* () { yield* validateRelayConfigPayload(payload); @@ -499,16 +499,19 @@ const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(fu CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(endpointRuntimeJson), ); - const registered = yield* registerManagedCloudTunnelRecovery().pipe( + yield* registerManagedCloudTunnelRecovery().pipe( + Effect.retry({ + times: 2, + while: (error) => + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + }), Effect.catch((cause) => Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { cause, - }).pipe(Effect.as(false)), + }), ), ); - if (!registered && options?.requestRecovery !== false) { - yield* dependencies.endpointRuntime.requestRecovery(payload.endpointRuntime); - } } else { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); } @@ -655,7 +658,7 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi cloudMintPublicKey: link.cloudMintPublicKey, endpointRuntime: link.endpointRuntime, }, - { requestRecovery: false, lockHeld: true }, + { lockHeld: true }, ); }, Effect.catchIf( diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 13867bf25d74..80b01bd9122d 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -2607,14 +2607,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); - assert.deepEqual(requestedRecoveryConfigs, [ - { - providerKind: "cloudflare_tunnel", - connectorToken: "connector-token", - tunnelId: "tunnel-id", - tunnelName: "tunnel-name", - }, - ]); + assert.deepEqual(requestedRecoveryConfigs, []); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index 843604e5c4eb..5484b3ae519d 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -103,10 +103,12 @@ describe("ManagedEndpointAllocations", () => { const query = new PgDialect().sqlToQuery(condition as never); expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"'); expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); expect(query.sql).toContain("for update"); expect(query.params).toContain("tunnel-1"); expect(query.params).toContain("public-key"); + expect(query.params).toContain("cloudflare_tunnel"); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index bd764071c308..c26b71d285c1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -400,6 +400,7 @@ export const make = Effect.gen(function* () { eq(relayEnvironmentLinks.userId, input.userId), eq(relayEnvironmentLinks.environmentId, input.environmentId), eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), isNull(relayEnvironmentLinks.revokedAt), ), ) diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index 4f1c15ff6bcf..ca0a321b4ded 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -571,7 +571,19 @@ describe("relay managed tunnel recovery", () => { ); }); - it.effect("removes a recovered tunnel when its link disappears before registration", () => { + it.effect.each([ + { state: "removed", currentLink: null }, + { + state: "publish-only", + currentLink: { + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }, + }, + ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => { let lookups = 0; const cleaned: Array = []; const target = { @@ -601,7 +613,8 @@ describe("relay managed tunnel recovery", () => { Effect.provide( Layer.merge( relayUnlinkTestLayer({ - getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + getForUser: () => + Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)), provision: () => Effect.succeed({ endpoint: linkedEnvironmentRecord.endpoint, diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index ce568652efd9..df1d1661279b 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -621,10 +621,14 @@ export const recoverEnvironmentTunnelRecord = Effect.fn( if (!enabled) { const owner = { userId: input.userId, environmentId: input.environmentId }; const target = yield* managedEndpointProvider.prepareDeprovision(owner); - if (target !== null && (yield* links.getForUser(input)) === null) { + const currentLink = target === null ? null : yield* links.getForUser(input); + if ( + target !== null && + (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel") + ) { yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( Effect.catch((cause) => - Effect.logWarning("Failed to clean up a tunnel after its link was removed", { + Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", { userId: input.userId, environmentId: input.environmentId, cause,