diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index b45b5099252a..cb2441095f5d 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -6,9 +6,11 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -60,6 +62,7 @@ function makeHandle(input: { readonly onKill: () => void; readonly isRunning?: () => boolean; readonly exitCode?: Effect.Effect; + readonly output?: Stream.Stream; }) { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(input.pid), @@ -73,13 +76,70 @@ function makeHandle(input: { stdin: Sink.drain, stdout: Stream.empty, stderr: Stream.empty, - all: Stream.empty, + all: input.output ?? Stream.empty, getInputFd: () => Sink.drain, getOutputFd: () => Stream.empty, }); } describe("CloudManagedEndpointRuntime", () => { + it("retries connector startup failures but stops for unsupported runtimes", () => { + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "not-installed", + reason: "The relay client is not installed.", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "spawn-failed", + reason: "spawn failed", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "unsupported-platform", + reason: "Relay client is unsupported on linux-arm.", + }), + ).toBe(false); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }), + ).toBe(false); + }); + + it.effect("serializes updates to persisted cloud link state", () => + Effect.gen(function* () { + const firstEntered = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const secondEntered = yield* Deferred.make(); + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + + const first = yield* runtime + .withLinkStateLock( + Deferred.succeed(firstEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + ), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(firstEntered); + + const second = yield* runtime + .withLinkStateLock(Deferred.succeed(secondEntered, undefined)) + .pipe(Effect.forkChild); + expect(yield* Deferred.isDone(secondEntered)).toBe(false); + + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Deferred.isDone(secondEntered)).toBe(true); + }), + ); + it("classifies Cloudflare connection and warning output", () => { expect( ManagedEndpointRuntime.classifyRelayClientOutput( @@ -107,6 +167,125 @@ describe("CloudManagedEndpointRuntime", () => { ).toBe("warning"); }); + it("recognizes tunnel authorization failures without matching ordinary transport errors", () => { + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', + ), + ).toBe(false); + }); + + it.effect("keeps recovery requests sent before the server starts consuming them", () => + Effect.gen(function* () { + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "tunnel-1", + }; + + yield* runtime.requestRecovery(config); + + expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config); + }), + ); + + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const firstBatchObserved = yield* Deferred.make(); + const secondBatchObserved = yield* Deferred.make(); + const recoveryRequested = yield* Deferred.make(); + const recoveryRetried = yield* Deferred.make(); + let recoveryRequestCount = 0; + const spawned: Array = []; + const encoder = new TextEncoder(); + const connectorOutput = Stream.fromQueue(output).pipe( + Stream.tap((chunk) => { + const line = new TextDecoder().decode(chunk); + if (line === "first checkpoint\n") { + return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid); + } + if (line === "second checkpoint\n") { + return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid); + } + return Effect.void; + }), + ); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const pid = 600; + spawned.push(pid); + const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "deleted-tunnel", + }; + const rejectedLine = + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Failed to get tunnel" connIndex=0\n'; + + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => { + recoveryRequestCount += 1; + return Deferred.succeed( + recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried, + requested, + ).pipe(Effect.asVoid); + }), + Effect.forkChild, + ); + yield* runtime.applyConfig(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3))); + yield* Queue.offer(output, encoder.encode("first checkpoint\n")); + yield* Deferred.await(firstBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer( + output, + encoder.encode( + "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" + + rejectedLine.repeat(3), + ), + ); + yield* Queue.offer(output, encoder.encode("second checkpoint\n")); + yield* Deferred.await(secondBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer(output, encoder.encode(rejectedLine)); + + expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4))); + + expect(yield* Deferred.await(recoveryRetried)).toEqual(config); + expect(spawned).toEqual([600]); + }), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; @@ -388,6 +567,7 @@ describe("CloudManagedEndpointRuntime", () => { expect(status).toEqual({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "The relay client is not installed.", }); expect(spawn).not.toHaveBeenCalled(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 89c0a23783c0..7695327da63b 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -36,6 +37,7 @@ export type CloudManagedEndpointRuntimeStatus = | { readonly status: "failed"; readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; + readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed"; readonly reason: string; readonly tunnelId?: string; readonly tunnelName?: string; @@ -58,6 +60,9 @@ export class CloudManagedEndpointRuntime extends Context.Service< readonly applyConfig: ( config: RelayManagedEndpointRuntimeConfig | null, ) => Effect.Effect; + readonly recoveryRequests: Stream.Stream; + readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -68,6 +73,9 @@ interface ActiveConnector { readonly config: RelayManagedEndpointRuntimeConfig; } +// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. +const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; + export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" { if (/\bRegistered tunnel connection\b/iu.test(line)) { return "connected"; @@ -78,6 +86,26 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +export function isRejectedRelayClientTunnelOutput(line: string): boolean { + return ( + /\bRegister tunnel error from server side\b/iu.test(line) && + /\bUnauthorized:\s*(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)\b/iu.test( + line, + ) + ); +} + +/** Connector startup failures can clear after installation or a later spawn attempt. */ +export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { + if (typeof status !== "object" || status === null || !("status" in status)) { + return false; + } + if (status.status !== "failed" || !("failure" in status)) { + return false; + } + return status.failure === "not-installed" || status.failure === "spawn-failed"; +} + function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { return JSON.stringify({ providerKind: config.providerKind, @@ -104,7 +132,9 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); + const recoveryRequests = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); + const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; const stopActive = Effect.gen(function* () { @@ -144,6 +174,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); + yield* Queue.offer(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -151,8 +182,10 @@ export const make = Effect.gen(function* () { Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), ); - const observeConnectorOutput = (connector: ActiveConnector) => - connector.child.all.pipe( + const observeConnectorOutput = (connector: ActiveConnector) => { + let rejectedRegistrations = 0; + + return connector.child.all.pipe( Stream.decodeText(), Stream.splitLines, Stream.map((line) => line.trim()), @@ -167,8 +200,22 @@ export const make = Effect.gen(function* () { }; switch (classifyRelayClientOutput(line)) { case "connected": + rejectedRegistrations = 0; return Effect.logInfo("Relay client tunnel connection registered", attributes); case "warning": + if (isRejectedRelayClientTunnelOutput(line)) { + rejectedRegistrations += 1; + if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) { + rejectedRegistrations = 0; + return Effect.logWarning( + "Relay client tunnel was rejected; requesting recovery", + attributes, + ).pipe( + Effect.andThen(Queue.offer(recoveryRequests, connector.config)), + Effect.asVoid, + ); + } + } return Effect.logWarning("Relay client reported a transport warning", attributes); case "debug": return Effect.logDebug("Relay client output", attributes); @@ -183,6 +230,7 @@ export const make = Effect.gen(function* () { }), ), ); + }; reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) { if (!config || config.providerKind !== "cloudflare_tunnel") { @@ -214,6 +262,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed", reason: executable.status === "unsupported" ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.` @@ -256,6 +305,7 @@ export const make = Effect.gen(function* () { Effect.as({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: String(cause), ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -290,6 +340,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: "Relay client did not start.", ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -305,6 +356,9 @@ export const make = Effect.gen(function* () { const runtime = CloudManagedEndpointRuntime.of({ applyConfig, + recoveryRequests: Stream.fromQueue(recoveryRequests), + requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), + withLinkStateLock: linkStateSemaphore.withPermits(1), }); const initialConfig = yield* readRuntimeConfig.pipe( diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 0f24e6f34176..6e500aec5816 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -6,7 +6,9 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; import * as Tracer from "effect/Tracer"; +import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, @@ -29,14 +31,25 @@ import { import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; -import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts"; +import { + RelayManagedEndpointRecoveryRegistrationRequest, + type RelayLinkProofRequest, +} from "@t3tools/contracts/relay"; +import { + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeRuntimeConfig, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_URL_SECRET, +} from "./config.ts"; import { consumeCloudReplayGuards, isSupportedLinkProviderKind, linkProofScopes, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./http.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -54,6 +67,9 @@ const storeFailure = (tag: "AlreadyExists" | "PermissionDenied") => }); const unusedSecretStoreOperation = () => Effect.die("unused secret-store operation"); +const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( + Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), +); function makeSecretStore( create: ServerSecretStore.ServerSecretStore["Service"]["create"], @@ -209,6 +225,9 @@ describe("reconcileDesiredCloudLink", () => { ManagedEndpointRuntime.CloudManagedEndpointRuntime, ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: unusedSecretStoreOperation, + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -251,7 +270,10 @@ describe("releaseManagedTunnelOnShutdown", () => { Effect.sync(() => { values.set(name, value); }), - create: unusedSecretStoreOperation, + create: (name, value) => + Effect.sync(() => { + values.set(name, value); + }), getOrCreateRandom: unusedSecretStoreOperation, remove: (name) => Effect.sync(() => { @@ -303,10 +325,19 @@ describe("releaseManagedTunnelOnShutdown", () => { applyConfig: (config) => Effect.sync(() => { harness.applyConfigCalls.push(config); - return { - status: "disabled", - } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus; + return config === null + ? ({ + status: "disabled", + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus) + : ({ + status: "running", + providerKind: "cloudflare_tunnel", + pid: 123, + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus); }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, }), ), Effect.provideService( @@ -579,6 +610,217 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ); }); + + it.effect("registers an existing tunnel without provisioning or restarting it", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe( + "https://relay.example.test/v1/environments/env_123/tunnel/recovery", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + const body = requests[0]?.body; + expect(body?._tag).toBe("Uint8Array"); + if (body?._tag === "Uint8Array") { + expect( + yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), + ).toMatchObject({ + cloudUserId: "user-123", + tunnelId: "existing-tunnel", + }); + } + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("does not register recovery without a recorded tunnel ID", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery()).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("recovers a web-linked tunnel with its environment credential", () => { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel", + connectorToken: "new-token", + tunnelId: "new-tunnel", + } as const; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel"); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + expect(applyConfigCalls).toEqual([nextConfig]); + expect( + Option.getOrNull( + decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))), + ), + ).toEqual(nextConfig); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + }), + ); + }); + + it.effect("does not recover an environment without a managed tunnel credential", () => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect( + yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", { + providerKind: "cloudflare_tunnel", + connectorToken: "old-token", + tunnelId: "old-tunnel", + }), + ).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect.each([ + { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 403, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 409, errorTag: "EnvironmentHttpConflictError" }, + ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773")); + + expect(error._tag).toBe(errorTag); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({}, { status }), + }), + ); + }); + + it.effect("keeps a tunnel configuration replaced during recovery", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const freshConfig = new TextEncoder().encode("fresh-config"); + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); + return Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + }, + }); + }, + }), + ); + }); }); describe("link proof provider kinds", () => { diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 29fdfe8ece2f..2adc803bb4db 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -28,6 +28,9 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, + RelayManagedEndpointRecoveryResponse, + type RelayManagedEndpointRuntimeConfig, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -36,6 +39,7 @@ import { RELAY_HEALTH_REQUEST_TYP, RELAY_HEALTH_RESPONSE_TYP, RELAY_LINK_PROOF_TYP, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, RELAY_MINT_REQUEST_TYP, RELAY_MINT_RESPONSE_TYP, signRelayJwt, @@ -70,6 +74,7 @@ import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, + decodeRuntimeConfig, encodeEndpointRuntimeConfigJson, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, @@ -453,46 +458,66 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest, + options?: { readonly lockHeld?: boolean }, ) { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, - }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, - ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, + const apply = Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, }); - } + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + const ok = + endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; + if (!ok) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, + }); + } - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), - ); - yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey)); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), ); - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), + ); + yield* registerManagedCloudTunnelRecovery().pipe( + Effect.retry({ + times: 2, + while: (error) => + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + }), + Effect.catch((cause) => + Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }), + ), + ); + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }); + return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply); }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( @@ -526,13 +551,29 @@ const relayClientRequest = ( HttpClientRequest.bearerToken(input.token), HttpClientRequest.bodyJson(input.payload), Effect.flatMap(dependencies.httpClient.execute), - Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status === 401 || response.status === 403) { + return yield* new EnvironmentHttpUnauthorizedError({ + message: "T3 Connect rejected the stored environment credential.", + }); + } + if (response.status === 409) { + return yield* new EnvironmentHttpConflictError({ + message: "T3 Connect rejected the current environment link.", + }); + } + return yield* HttpClientResponse.filterStatusOk(response); + }), + ), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), - Effect.mapError( - (cause) => - new EnvironmentHttpInternalServerError({ - message: `T3 Connect relay request failed: ${String(cause)}`, - }), + Effect.mapError((cause) => + cause._tag === "EnvironmentHttpUnauthorizedError" || + cause._tag === "EnvironmentHttpConflictError" + ? cause + : new EnvironmentHttpInternalServerError({ + message: `T3 Connect relay request failed: ${String(cause)}`, + }), ), withRelayClientTracing, ); @@ -607,14 +648,18 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi schema: RelayEnvironmentLinkResponse, }); yield* setCliDesiredCloudLink(true, mode); - return yield* applyCloudRelayConfig(dependencies, { - relayUrl, - relayIssuer: link.relayIssuer, - cloudUserId: link.cloudUserId, - environmentCredential: link.environmentCredential, - cloudMintPublicKey: link.cloudMintPublicKey, - endpointRuntime: link.endpointRuntime, - }); + return yield* applyCloudRelayConfig( + dependencies, + { + relayUrl, + relayIssuer: link.relayIssuer, + cloudUserId: link.cloudUserId, + environmentCredential: link.environmentCredential, + cloudMintPublicKey: link.cloudMintPublicKey, + endpointRuntime: link.endpointRuntime, + }, + { lockHeld: true }, + ); }, Effect.catchIf( ServerSecretStore.isSecretStoreError, @@ -631,7 +676,209 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")( function* (localOrigin: string) { - return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin); + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + reconcileDesiredCloudLinkWith(dependencies, localOrigin), + ); + }, +); + +type ManagedTunnelRecoveryProofInput = { + readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"]; + readonly cloudUserId: string; + readonly relayUrl: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +const makeManagedTunnelRecoveryProof = Effect.fn( + "environment.cloud.makeManagedTunnelRecoveryProof", +)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) { + const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets); + const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const claims = { + iss: `t3-env:${input.environmentId}`, + aud: normalizeRelayIssuer( + Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl, + ), + sub: input.environmentId, + jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)), + iat: issuedAt, + exp: issuedAt + 60, + environmentId: input.environmentId, + cloudUserId: input.cloudUserId, + }; + const payload = + input.action === "register" + ? { ...claims, action: "register" as const, tunnelId: input.tunnelId } + : { ...claims, action: "recover" as const, origin: input.origin }; + + return yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload, + }).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not sign the managed tunnel recovery request.", + }), + ), + ); +}); + +export const registerManagedCloudTunnelRecovery = Effect.fn( + "environment.cloud.registerManagedCloudTunnelRecovery", +)(function* () { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if (config?.providerKind !== "cloudflare_tunnel" || config.tunnelId === undefined) { + return false; + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "register", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + tunnelId: config.tunnelId, + }); + const registered = yield* relayClientRequest(dependencies, { + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: cloudUserIdValue, + tunnelId: config.tunnelId, + proof, + }, + schema: RelayOkResponse, + }); + return registered.ok; +}); + +export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( + function* (localOrigin: string, expectedConfig?: RelayManagedEndpointRuntimeConfig) { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + if (expectedConfig !== undefined) { + const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if ( + current === null || + current.providerKind !== expectedConfig.providerKind || + current.connectorToken !== expectedConfig.connectorToken || + current.tunnelId !== expectedConfig.tunnelId || + current.tunnelName !== expectedConfig.tunnelName + ) { + return false; + } + } + + const localUrl = yield* Effect.try({ + try: () => new URL(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + if (localUrl.origin !== localOrigin) { + return yield* new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }); + } + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const origin = { + localHttpHost: localUrl.hostname, + localHttpPort: endpointRequestPort(localUrl), + }; + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "recover", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + origin, + }); + const recovered = yield* relayClientRequest(dependencies, { + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: cloudUserIdValue, + origin, + proof, + }, + schema: RelayManagedEndpointRecoveryResponse, + }); + if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { + return yield* new EnvironmentHttpInternalServerError({ + message: "T3 Connect returned an unsupported managed tunnel configuration.", + }); + } + + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } + + const status = yield* dependencies.endpointRuntime.applyConfig(recovered.endpointRuntime); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + return true; + }), + ); }, ); @@ -788,21 +1035,25 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")( const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( function* (dependencies: CloudHttpDependencies) { yield* requireEnvironmentScope(AuthRelayWriteScope); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); - yield* Effect.all( - [ - dependencies.secrets.remove(CLOUD_LINKED_USER_ID), - dependencies.secrets.remove(RELAY_URL_SECRET), - dependencies.secrets.remove(RELAY_ISSUER_SECRET), - dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), - dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), - dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 7 }, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); + yield* Effect.all( + [ + dependencies.secrets.remove(CLOUD_LINKED_USER_ID), + dependencies.secrets.remove(RELAY_URL_SECRET), + dependencies.secrets.remove(RELAY_ISSUER_SECRET), + dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), + dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + ], + { concurrency: 7 }, + ); + yield* setCliDesiredCloudLink(false); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* setCliDesiredCloudLink(false); - return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }, Effect.catchIf( ServerSecretStore.isSecretStoreError, diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5e4f19172eff..80b01bd9122d 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -943,6 +943,9 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, ...options?.layers?.cloudManagedEndpointRuntime, }), ), @@ -2509,6 +2512,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("unlinks local cloud state and disables the managed endpoint runtime", () => Effect.gen(function* () { const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { @@ -2525,6 +2529,10 @@ it.layer(NodeServices.layer)("server router seam", (it) => { ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), }); }, + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), }, }, }); @@ -2599,6 +2607,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); + assert.deepEqual(requestedRecoveryConfigs, []); }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); @@ -2918,6 +2927,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { Effect.succeed({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "cloudflared missing", tunnelId: "tunnel-1", }), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 0a31bf376dae..1b2dc519867f 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,9 +1,13 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import * as Cause from "effect/Cause"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -92,6 +96,8 @@ import { connectHttpApiLayer, pendingServiceUpdateExists, reconcileDesiredCloudLink, + recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; @@ -581,10 +587,6 @@ export const makeServerLayer = Layer.unwrap( : Layer.empty; const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( Effect.gen(function* () { - if (!hasCloudPublicConfig) { - yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); - return; - } const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( Effect.timeout("10 seconds"), Effect.tap((released) => @@ -613,36 +615,107 @@ export const makeServerLayer = Layer.unwrap( if (!cleanupBeforeActivation) { yield* Effect.addFinalizer(() => releaseManagedTunnel); } - if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; + const localOrigin = `http://127.0.0.1:${address.port}`; + const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const recoveryLock = yield* Semaphore.make(1); + const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => + recoveryLock.withPermits(1)( + recoverManagedCloudTunnel(localOrigin, config).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError" && + (error._tag !== "EnvironmentCloudEndpointUnavailableError" || + CloudManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + )), + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + Effect.tap((recovered) => + recovered ? Effect.logInfo("T3 Connect managed tunnel recovered") : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the T3 Connect managed tunnel", { + cause, + }), + ), + ), + ); + yield* endpointRuntime.recoveryRequests.pipe( + Stream.runForEach(recoverManagedTunnel), + Effect.forkScoped, + ); // No settling delay before the first attempt: routes are already // serving by the time activation opens this gate (the startup // sequence awaits routesReady), and the retry schedule below // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( - Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const wantsCliLink = hasCloudPublicConfig + ? yield* CloudCliState.readCliDesiredCloudLink.pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to read the desired T3 Connect link", { cause }).pipe( + Effect.as(false), + ), ), - Schedule.upTo({ duration: "10 minutes" }), - ), - }), - Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), - Effect.catch((cause) => - Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, - }), + ) + : false; + const desiredCliLinkMode = wantsCliLink + ? yield* CloudCliState.readCliDesiredLinkMode + : null; + const registerManagedTunnel = registerManagedCloudTunnelRecovery().pipe( + Effect.tap((registered) => + registered + ? Effect.logInfo("T3 Connect managed tunnel recovery registered") + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to register T3 Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as(false)), ), ); + const registered = + desiredCliLinkMode === "publish_only" ? false : yield* registerManagedTunnel; + if (wantsCliLink && !registered) { + const reconciled = yield* reconcileDesiredCloudLink(localOrigin).pipe( + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.as(true), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { + cause, + }).pipe(Effect.as(false)), + ), + ); + if (reconciled && desiredCliLinkMode === "managed") { + yield* registerManagedTunnel; + } + } }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 6f796123e98b..0c7ed2ba3096 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -125,6 +125,27 @@ connector, and attempts to revoke the relay-side environment record. It retains authorization so `t3 connect link` can re-enable exposure without another browser flow. `t3 connect logout` performs the same cleanup and removes the stored CLI authorization. +### Managed tunnel lifecycle + +Every linked environment stores a relay-issued environment credential. When setup installs a tunnel +and when the server starts, the server uses that credential to register recovery support for the +existing tunnel. Registration only updates the relay database and does not call Cloudflare. +Healthy CLI links reuse the stored tunnel instead of provisioning it again. + +If the connector exits or repeatedly reports that Cloudflare rejected its tunnel, the server uses +the same environment credential to request a replacement. This also recovers a tunnel deleted +while a laptop was asleep, even when the connector keeps running. Environments linked through web +or mobile settings do not need a stored CLI credential. The relay keeps the existing hostname and +DNS record, so a replacement tunnel does not change the public endpoint. Each registration and +recovery request includes a short-lived host signature that binds the cloud user and the current +tunnel or local T3 server address. + +After a host registers recovery support, the existing five-minute maintenance job removes its +tunnel when Cloudflare reports that the tunnel has been down for at least five minutes. Tunnels +that never connected are removed when they are at least five minutes old. The job leaves older +hosts alone until they register recovery support, and it only removes tunnels that belong to its +own deployment stage. + The background service has an independent lifecycle. Connect setup may offer to install it, but logout leaves it running; manage it with `t3 service status`, `install`, `update`, and `uninstall`. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 5993fca5b352..67259f8cce26 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -229,6 +229,11 @@ works for a server that was wiped or is no longer reachable. Device-local connec controls remain in **Settings** → **Connections** on web and desktop or **Settings** → **Environments** on mobile. +If a linked environment stays offline for several minutes, T3 Connect removes its unused tunnel. +The environment stays linked to your account and keeps the same address. When the server starts +again or the computer wakes, T3 Connect creates a replacement tunnel automatically. You do not +need to pair it again. + ## Security Notes - Treat pairing URLs and pairing tokens like passwords. diff --git a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql new file mode 100644 index 000000000000..a9480ed4c93b --- /dev/null +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json new file mode 100644 index 000000000000..ce549ba1c6c0 --- /dev/null +++ b/infra/relay/migrations/postgres/20260825044249_managed_endpoint_recovery/snapshot.json @@ -0,0 +1,1542 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "7e85c554-d61a-4253-bd7b-17f92e98e665", + "prevIds": ["2374caff-40bf-423c-9255-55e76dddbc2a"], + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/deploymentConfig.test.ts b/infra/relay/src/deploymentConfig.test.ts index 44c7627a4daf..175e36ca527d 100644 --- a/infra/relay/src/deploymentConfig.test.ts +++ b/infra/relay/src/deploymentConfig.test.ts @@ -7,6 +7,7 @@ import { managedEndpointHostname, isManagedEndpointHostname, managedEndpointTunnelName, + managedEndpointTunnelNamePrefix, relayOwnsManagedEndpointZone, RelayPublicDomainLabelTooLongError, relayPublicDomainForStage, @@ -91,6 +92,9 @@ describe("managed endpoint names", () => { expect(managedEndpointTunnelName("dev_julius", hash)).toBe( "t3coderelay-managedendpoint-dev-julius-abcdef0123456789", ); + expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe( + "t3coderelay-managedendpoint-dev-julius-", + ); }); it("keeps the DNS label within the provider limit for long stage names", () => { diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index fe9d37b29988..1dab362a9f09 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi }; } +export function managedEndpointTunnelNamePrefix(stage: string): string { + return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`; +} + export function managedEndpointTunnelName(stage: string, hash: string): string { - return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`; + return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 7f536bafb375..31e6d27c8fcd 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -189,6 +189,7 @@ function makeAllocations( dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -197,7 +198,10 @@ function makeAllocations( recordTunnel: () => Effect.die("unused"), recordDns: () => Effect.die("unused"), markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -472,6 +476,7 @@ describe("EnvironmentConnector", () => { dnsRecordId: "dns-record-id", readyAt: null, updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }), }), ), diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index c0811e82d923..536c4e289695 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -137,7 +137,7 @@ function testLayer(input?: { }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { prepareDeprovision: () => Effect.succeed(null), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), provision: () => Effect.succeed({ @@ -243,6 +243,7 @@ describe("EnvironmentLinker", () => { deprovision: (input) => Effect.sync(() => { deprovisionedEnvironmentId = input.environmentId; + return true; }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ebf51de100c1..5484b3ae519d 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; @@ -10,17 +12,234 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("clears endpoint readiness only when the recorded tunnel changes", () => { + let updated: + | { + readonly tunnelId: string; + readonly readyAt: unknown; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { readonly tunnelId: string; readonly readyAt: unknown }) => { + updated = values; + return { + where: () => ({ + returning: () => Effect.succeed([{ generation: 8 }]), + }), + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.recordTunnel({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + generation: 7, + }), + ).toBe(8); + expect(updated?.tunnelId).toBe("replacement-tunnel"); + const query = new PgDialect().sqlToQuery(updated?.readyAt as never); + expect(query.sql).toBe( + 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end', + ); + expect(query.params).toEqual(["replacement-tunnel"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("records recovery support and advances the allocation generation", () => { + let updated: + | { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + } + | undefined; + let condition: unknown; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + }) => { + updated = values; + return { + where: (where: unknown) => { + condition = where; + return { + returning: () => Effect.succeed([{ environmentId: "environment-1" }]), + }; + }, + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + environmentPublicKey: "public-key", + }), + ).toBe(true); + + expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); + expect(updated?.recoveryEnabledAt).toBeDefined(); + expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key"); + const query = new PgDialect().sqlToQuery(condition as never); + expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); + expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"'); + expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); + expect(query.sql).toContain("for update"); + expect(query.params).toContain("tunnel-1"); + expect(query.params).toContain("public-key"); + expect(query.params).toContain("cloudflare_tunnel"); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("rejects recovery when the tunnel or active link no longer matches", () => { + const fakeDb = { + update: () => ({ + set: () => ({ + where: () => ({ + returning: () => Effect.succeed([]), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "missing-tunnel", + environmentPublicKey: "public-key", + }), + ).toBe(false); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("returns recovery support with tunnel allocation lookups", () => { + const base = { + userId: "user-1", + hostname: "environment.example.test", + tunnelName: "managed-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-08-25T12:00:00.000Z", + updatedAt: "2026-08-25T12:00:00.000Z", + generation: 1, + }; + const fakeDb = { + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + leftJoin: () => ({ + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + recoveryEnvironmentPublicKey: null, + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-3", + tunnelId: "tunnel-3", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "old-key", + linkedEnvironmentPublicKey: "new-key", + }, + ]), + }), + }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.listByTunnelNames([ + "first-tunnel", + "second-tunnel", + "third-tunnel", + ]); + + expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ + ["tunnel-1", true], + ["tunnel-2", false], + ["tunnel-3", false], + ]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("skips the database for an empty tunnel lookup", () => + Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect(yield* allocations.listByTunnelNames([])).toEqual([]); + }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), + ); + + it.effect("splits large tunnel lookups into bounded database queries", () => { + const batchSizes: number[] = []; + const fakeDb = { + select: () => ({ + from: () => ({ + leftJoin: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`); + expect(yield* allocations.listByTunnelNames(names)).toEqual([]); + expect(batchSizes).toEqual([500, 500, 1]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { - let claimedAt: string | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly updatedAt: string }) => { - claimedAt = values.updatedAt; + set: (_values: { readonly updatedAt: string }) => { return { where: () => ({ - returning: () => Effect.succeed([{ userId: "user-1" }]), + returning: () => Effect.succeed([{ generation: 8 }]), }), }; }, @@ -33,14 +252,58 @@ describe("ManagedEndpointAllocations", () => { const generation = yield* allocations.claimDeprovision({ userId: "user-1", environmentId: "environment-1", - updatedAt: "captured-generation", + generation: 7, }); - expect(generation).toBe(claimedAt); + expect(generation).toBe(8); expect(generation).not.toBeNull(); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); + it.effect("holds the claimed allocation row while deleting its tunnel", () => { + const operations: string[] = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => { + operations.push("transaction"); + }).pipe(Effect.andThen(effect)), + }, + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => ({ + for: (strength: string) => + Effect.sync(() => { + operations.push(`lock:${strength}`); + return [{ generation: 7 }]; + }), + }), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.withClaimedTunnel( + { + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + generation: 7, + }, + Effect.sync(() => { + operations.push("delete"); + return true; + }), + ); + + expect(Option.getOrNull(result)).toBe(true); + expect(operations).toEqual(["transaction", "lock:update", "delete"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("does not remove an allocation superseded after a deprovision claim", () => { const fakeDb = { delete: (table: unknown) => { @@ -59,7 +322,7 @@ describe("ManagedEndpointAllocations", () => { yield* allocations.removeClaimed({ userId: "user-1", environmentId: "environment-1", - updatedAt: "outdated-claim-generation", + generation: 7, }), ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index 4320eeea3b72..c26b71d285c1 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,14 +1,17 @@ import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq } from "drizzle-orm"; +import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm"; +import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import { isSqlError } from "effect/unstable/sql/SqlError"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; -import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface ManagedEndpointAllocation { readonly userId: string; @@ -18,13 +21,16 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; - /** - * Doubles as the allocation's generation marker: every mutation rewrites it, - * so `claimRelease` can detect a provision that raced a release. - */ readonly updatedAt: string; + readonly generation: number; } +export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { + readonly recoveryEnabled: boolean; +} + +export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -50,6 +56,9 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "record-tunnel", "record-dns", "mark-ready", + "enable-recovery", + "list-tunnels", + "lock-tunnel", "claim-release", "claim-deprovision", "remove", @@ -82,23 +91,36 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; + readonly generation: number; } interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { readonly dnsRecordId: string; + readonly tunnelId: string; + readonly generation: number; +} + +interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly generation: number; } interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; - readonly updatedAt: string; + readonly generation: number; +} + +interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly environmentPublicKey: string; } interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } export class ManagedEndpointAllocations extends Context.Service< @@ -112,23 +134,36 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly markReady: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: MarkManagedEndpointReadyInput, + ) => Effect.Effect; + readonly enableRecovery: ( + input: EnableManagedEndpointRecoveryInput, + ) => Effect.Effect; + readonly listByTunnelNames: ( + tunnelNames: ReadonlyArray, + ) => Effect.Effect< + ReadonlyArray, + ManagedEndpointAllocationPersistenceError + >; /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the - * caller loaded. A concurrent provision rewrites `updatedAt` when it + * caller loaded. A concurrent provision increments `generation` when it * records its tunnel, which makes a stale claim fail and keeps the freshly * issued tunnel alive. */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; + readonly withClaimedTunnel: ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ) => Effect.Effect, E | ManagedEndpointAllocationPersistenceError, R>; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -138,7 +173,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimDeprovision: ( input: ClaimManagedEndpointDeprovisionInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly remove: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; @@ -157,6 +192,7 @@ const allocationSelection = { dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, updatedAt: relayManagedEndpointAllocations.updatedAt, + generation: relayManagedEndpointAllocations.generation, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -248,14 +284,23 @@ export const make = Effect.gen(function* () { recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* ( input: RecordManagedEndpointTunnelInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, + readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -270,14 +315,23 @@ export const make = Effect.gen(function* () { recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* ( input: RecordManagedEndpointDnsInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -290,17 +344,26 @@ export const make = Effect.gen(function* () { ); }), markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* ( - input: ManagedEndpointAllocationKey, + input: MarkManagedEndpointReadyInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ readyAt: now, updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -312,6 +375,123 @@ export const make = Effect.gen(function* () { ), ); }), + enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( + input: EnableManagedEndpointRecoveryInput, + ) { + const now = DateTime.formatIso(yield* DateTime.now); + return yield* db + .update(relayManagedEndpointAllocations) + .set({ + recoveryEnabledAt: now, + recoveryEnvironmentPublicKey: input.environmentPublicKey, + updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + exists( + new QueryBuilder() + .select({ userId: relayEnvironmentLinks.userId }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .for("update"), + ), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) + .pipe( + Effect.map((rows) => rows.length > 0), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "enable-recovery", + stage: "database-request", + ...input, + cause, + }), + ), + ); + }), + listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")( + function* (tunnelNames: ReadonlyArray) { + if (tunnelNames.length === 0) { + return []; + } + const batches = Array.from( + { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) }, + (_, index) => + tunnelNames.slice( + index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + ), + ); + const results = yield* Effect.forEach( + batches, + (batch) => + db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + recoveryEnvironmentPublicKey: + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + }) + .from(relayManagedEndpointAllocations) + .leftJoin( + relayEnvironmentLinks, + and( + eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId), + eq( + relayEnvironmentLinks.environmentId, + relayManagedEndpointAllocations.environmentId, + ), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) + .pipe( + Effect.map((rows) => + rows.map( + ({ + recoveryEnabledAt, + recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey, + ...allocation + }) => ({ + ...allocation, + recoveryEnabled: + recoveryEnabledAt !== null && + recoveryEnvironmentPublicKey !== null && + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, + }), + ), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ), + { concurrency: 1 }, + ); + return results.flat(); + }, + ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { @@ -319,17 +499,18 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where( and( whereAllocation(input), eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -344,22 +525,64 @@ export const make = Effect.gen(function* () { ); return claimed; }), + withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")( + function* ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ): Effect.fn.Return, E | ManagedEndpointAllocationPersistenceError, R> { + const lockError = (cause: unknown) => + new ManagedEndpointAllocationPersistenceError({ + operation: "lock-tunnel", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + cause, + }); + return yield* db.$client + .withTransaction( + db + .select({ generation: relayManagedEndpointAllocations.generation }) + .from(relayManagedEndpointAllocations) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .limit(1) + .for("update") + .pipe( + Effect.mapError(lockError), + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed(Option.none()) + : effect.pipe(Effect.map(Option.some)), + ), + ), + ) + .pipe(Effect.mapError((cause) => (isSqlError(cause) ? lockError(cause) : cause))); + }, + ), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) - .set({ updatedAt: claimedAt }) + .set({ + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -371,7 +594,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* ( input: ManagedEndpointAllocationKey, @@ -399,7 +622,7 @@ export const make = Effect.gen(function* () { .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) .returning({ userId: relayManagedEndpointAllocations.userId }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 4d136658c8fd..c7ae9c0d1569 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -6,6 +6,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as RelayConfiguration from "../Config.ts"; @@ -33,7 +34,7 @@ const config = RelayConfiguration.RelayConfiguration.of({ }); interface TunnelCall { - readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete"; + readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete"; readonly input: unknown; } @@ -62,6 +63,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId: function makeTunnelClient(calls: TunnelCall[] = []) { return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.sync(() => { + calls.push({ operation: "get", input: tunnelId }); + return { + id: tunnelId, + name: "managed-tunnel", + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }; + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -91,6 +102,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) { function makePersistentTunnelClient(calls: TunnelCall[] = []) { let tunnel: { readonly id: string; readonly name: string } | null = null; return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + calls.push({ operation: "get", input: tunnelId }); + return tunnel === null + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ) + : Effect.succeed({ + ...tunnel, + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }); + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -159,6 +187,7 @@ function makeDnsClient( function makeAllocations(calls: AllocationCall[] = []) { const allocations = new Map(); + const recoveryEnabled = new Set(); let generation = 0; const mutate = ( key: string, @@ -168,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) { ) => { const allocation = allocations.get(key); if (allocation !== undefined) { - allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` }); + allocations.set(key, { + ...change(allocation), + generation: allocation.generation + 1, + updatedAt: `generation-${++generation}`, + }); } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -186,6 +219,7 @@ function makeAllocations(calls: AllocationCall[] = []) { dnsRecordId: null, readyAt: null, updatedAt: `generation-${++generation}`, + generation: 0, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -193,27 +227,61 @@ function makeAllocations(calls: AllocationCall[] = []) { recordTunnel: (input) => Effect.sync(() => { calls.push({ operation: "recordTunnel", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, + readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), recordDns: (input) => Effect.sync(() => { calls.push({ operation: "recordDns", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, dnsRecordId: input.dnsRecordId, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), markReady: (input) => Effect.sync(() => { calls.push({ operation: "markReady", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, readyAt: "2026-06-02T00:00:00.000Z", })); + return true; + }), + enableRecovery: (input) => + Effect.sync(() => { + const allocation = allocations.get(allocationKey(input)); + if (allocation?.tunnelId !== input.tunnelId) { + return false; + } + recoveryEnabled.add(allocationKey(input)); + return true; }), + listByTunnelNames: (tunnelNames) => + Effect.sync(() => + [...allocations.values()] + .filter((allocation) => tunnelNames.includes(allocation.tunnelName)) + .map((allocation) => ({ + ...allocation, + recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + })), + ), claimRelease: (input) => Effect.sync(() => { calls.push({ operation: "claimRelease", input }); @@ -221,22 +289,29 @@ function makeAllocations(calls: AllocationCall[] = []) { if ( allocation === undefined || allocation.tunnelId !== input.tunnelId || - allocation.updatedAt !== input.updatedAt + allocation.generation !== input.generation ) { - return false; + return null; } mutate(allocationKey(input), (current) => current); - return true; + return allocations.get(allocationKey(input))?.generation ?? null; + }), + withClaimedTunnel: (input, effect) => + Effect.suspend(() => { + const current = allocations.get(allocationKey(input)); + return current?.tunnelId === input.tunnelId && current.generation === input.generation + ? effect.pipe(Effect.map(Option.some)) + : Effect.succeed(Option.none()); }), claimDeprovision: (input) => Effect.sync(() => { calls.push({ operation: "claimDeprovision", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; }), remove: (input) => Effect.sync(() => { @@ -247,7 +322,7 @@ function makeAllocations(calls: AllocationCall[] = []) { Effect.sync(() => { calls.push({ operation: "removeClaimed", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return false; } allocations.delete(allocationKey(input)); @@ -306,6 +381,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"): describe("ManagedEndpointProvider", () => { it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => { const tunnelClient = { + get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }), list: () => Effect.succeed({ result: [] }), create: (request: { readonly name: string }) => Effect.succeed({ id: "tunnel-id", name: request.name }), @@ -731,7 +807,8 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.andThen(Effect.fail(failure))), deleteRecord: () => Effect.void, }); - const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations()); + const allocations = makeAllocations(); + const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -754,6 +831,10 @@ describe("ManagedEndpointProvider", () => { "createRecord", "updateRecord", ]); + expect(yield* allocations.get(request)).toMatchObject({ + tunnelId: "tunnel-id", + readyAt: "2026-06-02T00:00:00.000Z", + }); }).pipe(Effect.provide(layer)); }); @@ -914,7 +995,7 @@ describe("ManagedEndpointProvider", () => { // longer matches what the release loaded, so the claim fails. const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - claimRelease: () => Effect.succeed(false), + claimRelease: () => Effect.succeed(null), }); const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated); @@ -939,6 +1020,239 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not release a tunnel when the requested tunnel id is outdated", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + makeAllocations(), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("rejects a tunnel recorded after its allocation generation changed", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordTunnel: () => Effect.succeed(null), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create", "delete"]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordDns: () => Effect.succeed(null), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change tunnel ingress after another provision takes ownership", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: () => Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "configure-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change DNS after another provision takes ownership", () => { + const dnsCalls: DnsCall[] = []; + const allocations = makeAllocations(); + let lockCount = 0; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + ++lockCount === 1 + ? allocations.withClaimedTunnel(input, effect) + : Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + expect(dnsCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not mark a superseded tunnel allocation as ready", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + markReady: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "mark-allocation-ready", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { + const tunnelCalls: TunnelCall[] = []; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makePersistentTunnelClient(tunnelCalls), + get: (tunnelId) => + Effect.succeed({ + id: tunnelId, + name: expectedManagedTunnelName("env_ABC"), + status: "healthy", + connsInactiveAt: null, + }), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel when a new provision replaces the release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedGeneration) => + claimedGeneration === null + ? Effect.void + : allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid), + ), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 9a578874844e..507707213d5a 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare"; import * as Arr from "effect/Array"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Encoding from "effect/Encoding"; import * as Layer from "effect/Layer"; @@ -76,6 +77,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedErrorClass Effect.Effect; + }) => Effect.Effect; /** * Deletes the provisioned Cloudflare tunnel while keeping the allocation * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per @@ -166,16 +168,34 @@ export class ManagedEndpointProvider extends Context.Service< readonly release: (input: { readonly userId: string; readonly environmentId: string; + readonly expectedTunnelId?: string; + readonly expectedInactiveBefore?: string; + readonly expectedStatus?: "inactive" | "down"; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} -interface ManagedEndpointTunnel { +export interface ManagedEndpointTunnel { readonly id?: string | null; readonly name?: string | null; + readonly status?: string | null; + readonly createdAt?: string | null; + readonly connsInactiveAt?: string | null; +} + +export interface ManagedEndpointTunnelListRequest { + readonly isDeleted: false; + readonly name?: string; + readonly includePrefix?: string; + readonly status?: "inactive" | "down"; + readonly existedAt?: string; + readonly wasInactiveAt?: string; + readonly page?: number; + readonly perPage?: number; } const ManagedEndpointTunnelClientOperation = Schema.Literals([ + "get", "list", "create", "put-configuration", @@ -201,11 +221,18 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedErrorClass Effect.Effect< - { readonly result: ReadonlyArray }, + readonly get: ( + tunnelId: string, + ) => Effect.Effect; + readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< + { + readonly result: ReadonlyArray; + readonly resultInfo?: { + readonly page?: number | null; + readonly perPage?: number | null; + readonly totalCount?: number | null; + } | null; + }, ManagedEndpointTunnelClientError >; readonly create: (request: { @@ -333,17 +360,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean { ); } -function isNotFoundCause(cause: unknown): boolean { +export function isManagedEndpointNotFound(cause: unknown): boolean { if (typeof cause !== "object" || cause === null) { return false; } - if ("_tag" in cause && cause._tag === "NotFound") { + if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) { return true; } if ("status" in cause && cause.status === 404) { return true; } - return "cause" in cause && isNotFoundCause(cause.cause); + return "cause" in cause && isManagedEndpointNotFound(cause.cause); } type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError; @@ -355,9 +382,9 @@ const ignoreNotFound = ( Effect.asVoid, Effect.catchTags({ ManagedEndpointTunnelClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), }), ); @@ -399,7 +426,7 @@ export const make = Effect.gen(function* () { Effect.as(true), Effect.catchTags({ ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error), }), ); if (checkpointedRecordUpdated) { @@ -465,13 +492,13 @@ export const make = Effect.gen(function* () { const allocation = input.target === undefined ? yield* prepareDeprovision(input) : input.target; if (allocation === null) { - return; + return true; } - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimDeprovision({ userId: input.userId, environmentId: input.environmentId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -485,55 +512,86 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { - return; - } - const dnsRecordId = allocation.dnsRecordId; - if (dnsRecordId !== null) { - yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-dns-record", - dnsRecordId, - cause, - }), - ), - ); + if (claimedGeneration === null) { + return false; } const tunnelId = allocation.tunnelId; - if (tunnelId !== null) { - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-tunnel", - tunnelId, - cause, - }), - ), - ); + const deprovision = Effect.gen(function* () { + const dnsRecordId = allocation.dnsRecordId; + if (dnsRecordId !== null) { + yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-dns-record", + dnsRecordId, + cause, + }), + ), + ); + } + if (tunnelId !== null) { + yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-tunnel", + tunnelId, + cause, + }), + ), + ); + } + return yield* allocations + .removeClaimed({ + userId: input.userId, + environmentId: input.environmentId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "remove-allocation", + ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + }); + if (tunnelId === null) { + return yield* deprovision; } - yield* allocations - .removeClaimed({ - userId: input.userId, - environmentId: input.environmentId, - updatedAt: claimedAt, - }) + const removed = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + deprovision, + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "remove-allocation", - ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-deprovision", + tunnelId, + cause, + }), + ), + }), ); + return Option.getOrElse(removed, () => false); }), release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -554,19 +612,22 @@ export const make = Effect.gen(function* () { if (allocation === null || tunnelId === null) { return true; } + if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { + return false; + } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment - // restart) rewrites updatedAt when it records its tunnel, so a stale + // restart) increments the generation when it records its tunnel, so a stale // claim means the recorded tunnel may already back a fresh connector and // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimed = yield* allocations + const claimedGeneration = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, tunnelId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -579,10 +640,10 @@ export const make = Effect.gen(function* () { }), ), ); - if (!claimed) { + if (claimedGeneration === null) { return false; } - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => new ManagedEndpointDeprovisioningFailed({ @@ -593,6 +654,101 @@ export const make = Effect.gen(function* () { }), ), ); + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const expectedStatus = input.expectedStatus; + const inactiveBefore = input.expectedInactiveBefore; + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(inactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + + const released = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + Effect.gen(function* () { + const finalGeneration = yield* allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (finalGeneration === null) { + return false; + } + // Keep only the final delete inside the row lock so a concurrent + // provision cannot record this tunnel while Cloudflare removes it. + yield* deleteTunnel; + return true; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + }), + ); + return Option.getOrElse(released, () => false); + } + yield* deleteTunnel; // The recorded tunnelId is now stale, but the allocation row is left // untouched deliberately: connect/status authorization requires a fully // recorded allocation, and an offline environment must keep reporting @@ -684,13 +840,16 @@ export const make = Effect.gen(function* () { ); const { hostname, tunnelName } = allocation; - const tunnelResponse = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( + const selectedTunnel = yield* tunnels.list({ name: tunnelName, isDeleted: false }).pipe( Effect.map((tunnels) => tunnels.result), Effect.map(Arr.findFirst((tunnel) => tunnel.name === tunnelName)), Effect.flatMap( Option.match({ - onSome: (tunnel) => Effect.succeed(tunnel), - onNone: () => tunnels.create({ name: tunnelName, configSrc: "cloudflare" }), + onSome: (tunnel) => Effect.succeed({ tunnel, created: false }), + onNone: () => + tunnels + .create({ name: tunnelName, configSrc: "cloudflare" }) + .pipe(Effect.map((tunnel) => ({ tunnel, created: true }))), }), ), Effect.mapError( @@ -705,6 +864,7 @@ export const make = Effect.gen(function* () { }), ), ); + const tunnelResponse = selectedTunnel.tunnel; if (!tunnelResponse.id || tunnelResponse.name !== tunnelName) { return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, @@ -717,11 +877,12 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - yield* allocations + const tunnelGeneration = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: tunnel.id, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -737,31 +898,105 @@ export const make = Effect.gen(function* () { }), ), ); + if (tunnelGeneration === null) { + if (selectedTunnel.created) { + const current = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ); + if (current?.tunnelId !== tunnel.id) { + yield* ignoreNotFound(tunnels.delete(tunnel.id)).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to remove a tunnel that lost its allocation", { + tunnelId: tunnel.id, + tunnelName, + cause, + }), + ), + ); + } + } + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed during provisioning.", + }); + } - yield* tunnels - .putConfiguration(tunnel.id, { - ingress: [ - { - hostname, - service: formatOriginService(input.origin), - }, - { service: "http_status:404" }, - ], - }) + const configured = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + tunnels + .putConfiguration(tunnel.id, { + ingress: [ + { + hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ), + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "configure-tunnel", - hostname, - tunnelName, - tunnelId: tunnel.id, - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(configured)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed before its configuration was updated.", + }); + } const dnsRecord = { type: "CNAME", @@ -771,31 +1006,61 @@ export const make = Effect.gen(function* () { proxied: true, } as const; - const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "ensure-dns-record", + const recordedDns = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + Effect.gen(function* () { + const dnsRecordId = yield* ensureDnsRecord( hostname, - tunnelName, - tunnelId: tunnel.id, - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), - ); - yield* allocations - .recordDns({ - userId: input.userId, - environmentId: input.environmentId, - dnsRecordId, - }) - .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ + allocation.dnsRecordId, + dnsRecord, + ).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "ensure-dns-record", + hostname, + tunnelName, + tunnelId: tunnel.id, + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + const dnsGeneration = yield* allocations + .recordDns({ + userId: input.userId, + environmentId: input.environmentId, + dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause, + }), + ), + ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, stage: "record-dns", @@ -803,10 +1068,40 @@ export const make = Effect.gen(function* () { tunnelName, tunnelId: tunnel.id, dnsRecordId, - cause, - }), - ), + cause: "The tunnel allocation changed before its DNS record was saved.", + }); + } + return { dnsRecordId, dnsGeneration }; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(recordedDns)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause: "The tunnel allocation changed before its DNS record was updated.", + }); + } + const { dnsRecordId, dnsGeneration } = recordedDns.value; const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( @@ -823,10 +1118,12 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const ready = yield* allocations .markReady({ userId: input.userId, environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: dnsGeneration, }) .pipe( Effect.mapError( @@ -843,6 +1140,18 @@ export const make = Effect.gen(function* () { }), ), ); + if (!ready) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "mark-allocation-ready", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause: "The tunnel allocation changed before it became ready.", + }); + } return { endpoint: managedEndpointForHostname(hostname), @@ -865,16 +1174,28 @@ export const layerCloudflareBindings = ( alchemyRuntimeContext: Alchemy.BaseRuntimeContext, ) => layer.pipe( - Layer.provide( + Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ + get: (tunnelId) => + tunnelClient.get(tunnelId).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause, + }), + ), + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + ), list: (request) => tunnelClient.list(request).pipe( Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ operation: "list", - tunnelName: request.name, + ...(request.name === undefined ? {} : { tunnelName: request.name }), cause, }), ), diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts new file mode 100644 index 000000000000..335d58c3c217 --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -0,0 +1,593 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as TestClock from "effect/testing/TestClock"; + +import * as RelayConfiguration from "../Config.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts"; + +const NOW = "2026-08-25T12:00:00.000Z"; +const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds; +const PREFIX = "t3coderelay-managedendpoint-prod-"; + +function tunnel(input: { + readonly id: string; + readonly suffix: string; + readonly status: "down" | "inactive" | "healthy" | "degraded"; + readonly timestamp?: string | null; + readonly prefix?: string; +}): ManagedEndpointProvider.ManagedEndpointTunnel { + return { + id: input.id, + name: `${input.prefix ?? PREFIX}${input.suffix}`, + status: input.status, + ...(input.timestamp === undefined + ? {} + : input.status === "inactive" + ? { createdAt: input.timestamp } + : { connsInactiveAt: input.timestamp }), + }; +} + +function allocation(input: { + readonly tunnelId: string | null; + readonly recoveryEnabled: boolean; +}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { + return { + userId: "user-1", + environmentId: `environment-${input.tunnelId ?? "pending"}`, + hostname: `${input.tunnelId ?? "pending"}.example.test`, + tunnelId: input.tunnelId, + tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, + dnsRecordId: "dns-1", + readyAt: "2026-08-25T11:00:00.000Z", + updatedAt: "2026-08-25T11:00:00.000Z", + generation: 1, + recoveryEnabled: input.recoveryEnabled, + }; +} + +function harness(input?: { + readonly tunnels?: ReadonlyArray; + readonly allocations?: ReadonlyArray; + readonly namespace?: string; + readonly failTunnelId?: string; + readonly missingOnDeleteTunnelId?: string; + readonly missingOnGetTunnelId?: string; + readonly reserveOnGetTunnelId?: string; + readonly refreshedTunnels?: ReadonlyMap; + readonly skipTunnelId?: string; +}) { + const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; + const deleted: string[] = []; + const releases: Array< + Parameters[0] + > = []; + const remaining = [...(input?.tunnels ?? [])]; + const recorded = (input?.allocations ?? []).map((entry) => { + const matching = remaining.find((candidate) => candidate.id === entry.tunnelId); + return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; + }); + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + if (tunnelId === input?.missingOnGetTunnelId) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + const found = + input?.refreshedTunnels?.get(tunnelId) ?? + remaining.find((candidate) => candidate.id === tunnelId); + if (found === undefined) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") { + recorded.push({ + ...allocation({ tunnelId, recoveryEnabled: false }), + tunnelName: found.name, + }); + } + return Effect.succeed(found); + }), + list: (request) => + Effect.sync(() => { + listRequests.push(request); + const matching = remaining.filter((entry) => entry.status === request.status); + const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); + return { + result: matching.slice(start, start + (request.perPage ?? 100)), + resultInfo: { + page: request.page ?? 1, + perPage: request.perPage ?? 100, + totalCount: matching.length, + }, + }; + }), + create: () => Effect.die("unused"), + putConfiguration: () => Effect.die("unused"), + getToken: () => Effect.die("unused"), + delete: (tunnelId) => + tunnelId === input?.failTunnelId || tunnelId === input?.missingOnDeleteTunnelId + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause: + tunnelId === input?.missingOnDeleteTunnelId + ? { _tag: "NotFound" } + : "Cloudflare refused the deletion", + }), + ) + : Effect.sync(() => { + deleted.push(tunnelId); + const index = remaining.findIndex((candidate) => candidate.id === tunnelId); + if (index !== -1) { + remaining.splice(index, 1); + } + }), + }); + const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + get: () => Effect.die("unused"), + reserve: () => Effect.die("unused"), + recordTunnel: () => Effect.die("unused"), + recordDns: () => Effect.die("unused"), + markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: (tunnelNames) => + Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), + claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), + claimDeprovision: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + removeClaimed: () => Effect.die("unused"), + }); + const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({ + provision: () => Effect.die("unused"), + prepareDeprovision: () => Effect.die("unused"), + deprovision: () => Effect.die("unused"), + release: (request) => + Effect.sync(() => { + releases.push(request); + if (request.expectedTunnelId === input?.skipTunnelId) { + return false; + } + if (request.expectedTunnelId !== undefined) { + deleted.push(request.expectedTunnelId); + const index = remaining.findIndex( + (candidate) => candidate.id === request.expectedTunnelId, + ); + if (index !== -1) { + remaining.splice(index, 1); + } + } + return true; + }), + }); + const config = RelayConfiguration.RelayConfiguration.of({ + relayIssuer: "https://relay.example.test", + apns: { + environment: "sandbox", + teamId: "team-id", + keyId: "key-id", + privateKey: Redacted.make("private-key"), + bundleId: "com.t3tools.t3code.dev", + }, + apnsDeliveryJobSigningSecret: Redacted.make("job-secret"), + clerkSecretKey: Redacted.make("clerk-secret"), + clerkPublishableKey: "pk_test_test", + clerkJwtAudience: "t3-code-relay", + cloudMintPrivateKey: Redacted.make("cloud-private-key"), + cloudMintPublicKey: "cloud-public-key", + managedEndpointBaseDomain: "example.test", + managedEndpointNamespace: input?.namespace ?? "prod", + }); + + return { + listRequests, + deleted, + releases, + layer: ManagedEndpointReaper.layer.pipe( + Layer.provide( + Layer.mergeAll( + RelayConfiguration.layer(config), + ManagedEndpointProvider.layerTunnelClient(tunnelClient), + Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider), + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService), + ), + ), + ), + }; +} + +describe("ManagedEndpointReaper", () => { + it.effect("removes expired down and inactive tunnels from recoverable environments", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "down-1", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:00.000Z", + }), + tunnel({ + id: "inactive-1", + suffix: "bbbbbbbbbbbbbbbb", + status: "inactive", + timestamp: "2026-08-25T11:54:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "down-1", recoveryEnabled: true }), + allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 2, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual(["down-1", "inactive-1"]); + expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ + "down-1", + "inactive-1", + ]); + expect(state.listRequests).toEqual([ + { + isDeleted: false, + includePrefix: PREFIX, + status: "down", + existedAt: "2026-08-25T11:55:00.000Z", + wasInactiveAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + { + isDeleted: false, + includePrefix: PREFIX, + status: "inactive", + existedAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + ]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "recent", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:01.000Z", + }), + tunnel({ + id: "other-stage", + prefix: `${PREFIX}julius-`, + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "missing-time", + suffix: "cccccccccccccccc", + status: "inactive", + timestamp: null, + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 0, + deleted: 0, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps tunnels owned by environments that cannot recover yet", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "legacy", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 0, + skippedLegacy: 1, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("removes expired tunnels that no longer have an allocation", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "orphan", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["orphan"]); + expect(state.releases).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("removes an expired tunnel that was created but never recorded", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "unrecorded", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: null, recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(1); + expect(state.deleted).toEqual(["unrecorded"]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps an orphan tunnel that reconnects before deletion", () => { + const listed = tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }); + const state = harness({ + tunnels: [listed], + refreshedTunnels: new Map([ + [ + "reconnected", + tunnel({ + id: "reconnected", + suffix: "aaaaaaaaaaaaaaaa", + status: "healthy", + }), + ], + ]), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("treats an already deleted orphan tunnel as successfully removed", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "gone", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + missingOnDeleteTunnelId: "gone", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 1, + deleted: 1, + skippedLegacy: 0, + failed: 0, + }); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not delete an orphan tunnel reserved during the status check", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "reserved", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + reserveOnGetTunnelId: "reserved", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not count a tunnel that was replaced before its release", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "replaced", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })], + skipTunnelId: "replaced", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues after an orphan tunnel deletion fails", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "failed", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "next", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + failTunnelId: "failed", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 2, + deleted: 1, + skippedLegacy: 0, + failed: 1, + }); + expect(state.deleted).toEqual(["next"]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { + const entries = Array.from({ length: 101 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries.map((entry, index) => + allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 101, + deleted: 1, + skippedLegacy: 100, + failed: 0, + }); + expect(state.deleted).toEqual(["tunnel-100"]); + expect( + state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page), + ).toEqual([1, 2]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("collects every page before deletions shift Cloudflare pagination", () => { + const entries = Array.from({ length: 120 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries + .slice(50, 100) + .map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: false })), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + scanned: 120, + deleted: 70, + skippedLegacy: 50, + failed: 0, + }); + expect(state.deleted).toContain("tunnel-119"); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("limits each cleanup run to 100 tunnel deletions", () => { + const state = harness({ + tunnels: Array.from({ length: 105 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(100); + expect(state.deleted).toHaveLength(100); + }).pipe(Effect.provide(state.layer)); + }); +}); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts new file mode 100644 index 000000000000..0442d5e58a80 --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -0,0 +1,223 @@ +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import * as RelayConfiguration from "../Config.ts"; +import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; + +export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; +export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; +export const MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT = 100; + +export interface ManagedEndpointSweepResult { + readonly scanned: number; + readonly deleted: number; + readonly skippedLegacy: number; + readonly failed: number; +} + +export class ManagedEndpointReaper extends Context.Service< + ManagedEndpointReaper, + { + readonly sweep: Effect.Effect< + ManagedEndpointSweepResult, + | ManagedEndpointProvider.ManagedEndpointTunnelClientError + | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError + >; + } +>()("t3code-relay/environments/ManagedEndpointReaper") {} + +function isExpiredManagedTunnel(input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; +}): input is typeof input & { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; +} { + const { tunnel, status, prefix, cutoff } = input; + if ( + typeof tunnel.id !== "string" || + typeof tunnel.name !== "string" || + tunnel.status !== status || + !tunnel.name.startsWith(prefix) || + !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length)) + ) { + return false; + } + + const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; + if (typeof inactiveAt !== "string") { + return false; + } + const timestamp = DateTime.make(inactiveAt); + return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; +} + +export const make = Effect.gen(function* () { + const config = yield* RelayConfiguration.RelayConfiguration; + const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + + const deleteOrphan = Effect.fn("relay.managed_endpoint_reaper.delete_orphan")(function* (input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; + }) { + const current = yield* tunnels.get(input.tunnel.id).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(Option.none()) + : Effect.fail(error), + }), + ); + if (Option.isNone(current)) { + return true; + } + if (!isExpiredManagedTunnel({ ...input, tunnel: current.value })) { + return false; + } + if ( + (yield* allocations.listByTunnelNames([input.tunnel.name])).some( + (allocation) => allocation.tunnelId !== null, + ) + ) { + return false; + } + return yield* tunnels.delete(input.tunnel.id).pipe( + Effect.as(true), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + ManagedEndpointProvider.isManagedEndpointNotFound(error.cause) + ? Effect.succeed(true) + : Effect.fail(error), + }), + ); + }); + + const sweep = Effect.gen(function* () { + const namespace = config.managedEndpointNamespace; + if (!namespace) { + return { scanned: 0, deleted: 0, skippedLegacy: 0, failed: 0 }; + } + + const now = yield* DateTime.now; + const cutoff = DateTime.subtract(now, { minutes: MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES }); + const cutoffIso = DateTime.formatIso(cutoff); + const prefix = managedEndpointTunnelNamePrefix(namespace); + let deleted = 0; + let skippedLegacy = 0; + let failed = 0; + const expired: Array<{ + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + }> = []; + + for (const status of ["down", "inactive"] as const) { + let page = 1; + while (true) { + const response = yield* tunnels.list({ + isDeleted: false, + includePrefix: prefix, + status, + existedAt: cutoffIso, + ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}), + page, + perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, + }); + expired.push( + ...response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => ({ tunnel, status })), + ); + + const totalCount = response.resultInfo?.totalCount; + if ( + response.result.length === 0 || + (typeof totalCount === "number" + ? page * MANAGED_ENDPOINT_SWEEP_PAGE_SIZE >= totalCount + : response.result.length < MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) + ) { + break; + } + page += 1; + } + } + + const recorded = yield* allocations.listByTunnelNames(expired.map(({ tunnel }) => tunnel.name)); + const recordedByTunnelName = new Map( + recorded.map((allocation) => [allocation.tunnelName, allocation]), + ); + + for (const { tunnel, status } of expired) { + if (deleted >= MANAGED_ENDPOINT_SWEEP_DELETE_LIMIT) { + break; + } + const allocation = recordedByTunnelName.get(tunnel.name); + if ( + allocation !== undefined && + allocation.tunnelId !== null && + allocation.tunnelId !== tunnel.id + ) { + continue; + } + const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; + if (owner !== undefined && !owner.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + + const result = + owner === undefined + ? yield* deleteOrphan({ tunnel, status, prefix, cutoff }).pipe(Effect.result) + : yield* provider + .release({ + userId: owner.userId, + environmentId: owner.environmentId, + expectedTunnelId: tunnel.id, + expectedInactiveBefore: cutoffIso, + expectedStatus: status, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + return { scanned: expired.length, deleted, skippedLegacy, failed }; + }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep")); + + return ManagedEndpointReaper.of({ sweep }); +}); + +export const layer = Layer.effect(ManagedEndpointReaper, make); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index daf756a2b7cc..ca0a321b4ded 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -1,7 +1,9 @@ +import * as NodeCrypto from "node:crypto"; import { createClerkClient, verifyToken } from "@clerk/backend"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -16,6 +18,7 @@ import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; import { EnvironmentId } from "@t3tools/contracts"; import { RelayEnvironmentAuth } from "@t3tools/contracts/relay"; +import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { RELAY_REQUEST_DEADLINE_MS, @@ -23,16 +26,20 @@ import { relayDocsRedirectRoute, relayEnvironmentAuthLayer, relayNotFoundRoute, + recoverEnvironmentTunnelRecord, + registerEnvironmentTunnelRecovery, revokeEnvironmentLinkRecord, traceRelayHttpRequestWith, unlinkEnvironmentRecord, verifyRelayClientBearerToken, + verifyEnvironmentTunnelRecoveryProof, withoutCapturedParentSpan, } from "./Api.ts"; import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; vi.mock("@clerk/backend", () => ({ @@ -168,6 +175,8 @@ function relayUnlinkTestLayer(input?: { readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"]; readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; + readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; + readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; }) { return Layer.mergeAll( Layer.succeed( @@ -199,10 +208,10 @@ function relayUnlinkTestLayer(input?: { Layer.succeed( ManagedEndpointProvider.ManagedEndpointProvider, ManagedEndpointProvider.ManagedEndpointProvider.of({ - provision: () => Effect.die("unused provision"), + provision: input?.provision ?? (() => Effect.die("unused provision")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), - deprovision: input?.deprovision ?? (() => Effect.void), - release: () => Effect.die("unused release"), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), + release: input?.release ?? (() => Effect.die("unused release")), }), ), ); @@ -220,6 +229,419 @@ const linkedEnvironmentRecord = { linkedAt: "2026-07-28T00:00:00.000Z", } as const; +describe("relay managed tunnel recovery", () => { + it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "recovery-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "recover", + environmentId: "environment-1", + cloudUserId: "user-1", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + const request = { + action: "recover" as const, + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }; + + yield* verifyEnvironmentTunnelRecoveryProof(request); + + const wrongOwner = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }), + ); + expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" }); + + const wrongOrigin = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" }); + + const wrongAction = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + }), + ); + expect(wrongAction).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + + it.effect("registers recovery for an existing tunnel without provisioning it", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "existing-tunnel", + }), + ).toEqual({ ok: true }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "existing-tunnel", + environmentPublicKey: "public-key", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => Effect.die("registration must not provision a tunnel"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration for a different environment key", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + tunnelId: "existing-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration when the recorded tunnel changed", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "stale-tunnel", + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ), + ); + + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; + const runtime = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }; + + return Effect.gen(function* () { + expect( + yield* recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ + endpoint: linkedEnvironmentRecord.endpoint, + endpointRuntime: runtime, + }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + environmentPublicKey: "public-key", + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects a credential from a different environment owner", () => { + let provisioned = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(provisioned).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.sync(() => { + provisioned = true; + return { + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ); + }); + + it.effect("does not recover a publish-only environment", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.succeed({ + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ), + ); + + it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { + let recoveryEnabled = false; + const cleaned: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: { + httpBaseUrl: "https://different.example.test/", + wsBaseUrl: "wss://different.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.die("must keep the active allocation"), + deprovision: () => Effect.die("must keep the active link DNS"), + release: ({ expectedTunnelId }) => + Effect.sync(() => { + if (expectedTunnelId) { + cleaned.push(expectedTunnelId); + } + return true; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect.each([ + { state: "removed", currentLink: null }, + { + state: "publish-only", + currentLink: { + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }, + }, + ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => { + let lookups = 0; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "replacement-generation", + generation: 3, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ); + }); +}); + describe("relay environment unlink", () => { it.effect("revokes the link and its credentials in one database transaction", () => { const calls: Array = []; @@ -265,6 +687,7 @@ describe("relay environment unlink", () => { dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", updatedAt: "generation-before-unlink", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -313,6 +736,7 @@ describe("relay environment unlink", () => { Effect.sync(() => { expect(request.target).toBe(deprovisionTarget); calls.push("deprovision"); + return true; }), }), ), @@ -361,6 +785,7 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; }), }), ), @@ -388,6 +813,46 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; + }), + }), + ), + ); + }); + + it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => { + let lookups = 0; + const targets: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "tunnel-1", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + updatedAt: "original-generation", + generation: 1, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + expect( + yield* unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + }), + ).toBe(true); + expect(targets).toEqual([target, target]); + }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + targets.push(captured ?? undefined); + return targets.length > 1; }), }), ), diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 50bcff665a9b..df1d1661279b 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -46,10 +46,16 @@ import { RelayEnvironmentLinkLimitExceededError, RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, + type RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; -import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt"; +import { + normalizeRelayIssuer, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts"; import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts"; @@ -92,6 +98,10 @@ const relayCorsPreflightHeaders = { "access-control-max-age": "86400", } as const; +const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect( + RelayManagedEndpointRecoveryProofPayload, +); + const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler( (_request, response) => Effect.succeed( @@ -455,15 +465,185 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a // retry to finish cleanup after an earlier Cloudflare failure. - yield* managedEndpointProvider.deprovision({ + const deprovisioned = yield* managedEndpointProvider.deprovision({ userId: input.userId, environmentId: input.environmentId, target: deprovisionTarget, }); + if (!deprovisioned) { + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); + if (retryTarget !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + } + } return unlinked; }, ); +type EnvironmentTunnelRecoveryProofInput = { + readonly proof: string; + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; +} & ( + | { readonly action: "register"; readonly tunnelId: string } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +export const verifyEnvironmentTunnelRecoveryProof = Effect.fn( + "relay.api.server.verifyEnvironmentTunnelRecoveryProof", +)(function* (input: EnvironmentTunnelRecoveryProofInput) { + const config = yield* RelayConfiguration.RelayConfiguration; + const now = yield* DateTime.now; + const verified = yield* verifyRelayJwt({ + publicKey: input.environmentPublicKey, + token: input.proof, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + issuer: `t3-env:${input.environmentId}`, + audience: normalizeRelayIssuer(config.relayIssuer), + nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), + }).pipe( + Effect.flatMap(decodeManagedTunnelRecoveryProof), + Effect.mapError(() => new HttpApiError.Unauthorized({})), + ); + + if ( + verified.environmentId !== input.environmentId || + verified.sub !== input.environmentId || + verified.cloudUserId !== input.userId || + verified.action !== input.action + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (input.action === "register") { + if (verified.action !== "register" || verified.tunnelId !== input.tunnelId) { + return yield* new HttpApiError.Unauthorized({}); + } + return; + } + if ( + verified.action !== "recover" || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { + return yield* new HttpApiError.Unauthorized({}); + } +}); + +export const registerEnvironmentTunnelRecovery = Effect.fn( + "relay.api.server.registerEnvironmentTunnelRecovery", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly tunnelId: string; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (!(yield* allocations.enableRecovery(input))) { + return yield* new HttpApiError.Unauthorized({}); + } + return { ok: true }; +}); + +export const recoverEnvironmentTunnelRecord = Effect.fn( + "relay.api.server.recoverEnvironmentTunnelRecord", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + const recovered = yield* managedEndpointProvider.provision({ + userId: input.userId, + environmentId: input.environmentId, + origin: input.origin, + }); + const recoveredTunnelId = recovered.runtime.tunnelId; + if ( + recoveredTunnelId === undefined || + recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || + recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl + ) { + if (recoveredTunnelId !== undefined) { + yield* managedEndpointProvider + .release({ + userId: input.userId, + environmentId: input.environmentId, + expectedTunnelId: recoveredTunnelId, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } + + const enabled = yield* allocations.enableRecovery({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + environmentPublicKey: input.environmentPublicKey, + }); + if (!enabled) { + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); + const currentLink = target === null ? null : yield* links.getForUser(input); + if ( + target !== null && + (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel") + ) { + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", { + userId: input.userId, + environmentId: input.environmentId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } + return { + endpoint: recovered.endpoint, + endpointRuntime: recovered.runtime, + }; +}); + export const mobileApi = HttpApiBuilder.group( RelayApi, "mobile", @@ -856,7 +1036,7 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; - return handlers.handle( + const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( function* (args) { @@ -984,6 +1164,72 @@ export const serverApi = HttpApiBuilder.group( mapRelayCommonApiErrors("not_authorized"), ), ); + + return activityHandlers + .handle( + "registerManagedEndpointRecovery", + Effect.fn("relay.api.server.registerManagedEndpointRecovery")(function* ({ + params, + payload, + }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + }); + yield* appendRelayCredentialResponseHeaders; + return yield* registerEnvironmentTunnelRecovery({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + }); + }, mapRelayCommonApiErrors("not_authorized")), + ) + .handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "recover", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index 61b72f2df868..88196edc4fd6 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -93,6 +93,9 @@ export const relayManagedEndpointAllocations = pgTable( tunnelName: text("tunnel_name").notNull(), dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), + recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + recoveryEnvironmentPublicKey: text("recovery_environment_public_key"), + generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index 77dfd845c5bc..fe9c085fdea8 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -2,6 +2,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle"; import * as Config from "effect/Config"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -55,6 +56,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts"; import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts"; import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"; import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; @@ -195,7 +197,9 @@ export const ApiLive = Api.make( Layer.provideMerge(AgentActivityPublisher.layer), Layer.provideMerge(EnvironmentConnector.layer), Layer.provideMerge(EnvironmentLinker.layer), - Layer.provideMerge(EnvironmentPublishSignatures.layer), + Layer.provideMerge( + Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer), + ), Layer.provideMerge( ManagedEndpointProvider.layerCloudflareBindings( managedEndpointTunnelBinding, @@ -261,22 +265,42 @@ export const ApiLive = Api.make( ); yield* Cloudflare.Workers.cron("*/5 * * * *", () => - DpopProofs.DpopProofReplay.pipe( - Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), - // Terminal thread rows are kept briefly so finished agents show as - // Done/Failed in the Live Activity; sweep them once they age out. - Effect.andThen( - Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( - Effect.flatMap(([activityRows, now]) => - activityRows.pruneTerminal({ - updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), - }), + Effect.all( + [ + DpopProofs.DpopProofReplay.pipe( + Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), + // Keep completed thread rows long enough to show their final state. + Effect.andThen( + Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( + Effect.flatMap(([activityRows, now]) => + activityRows.pruneTerminal({ + updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), + }), + ), + ), + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), - ), - Effect.withSpan("relay.cron.prune_expired_state"), - Effect.provide(runtimeLayer), - ), + ManagedEndpointReaper.ManagedEndpointReaper.pipe( + Effect.flatMap((reaper) => reaper.sweep), + Effect.tap((result) => + result.scanned > 0 + ? Effect.logInfo("Finished managed tunnel cleanup", result) + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + ), + ), + ], + { concurrency: 2, discard: true }, + ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)), ); const fetch = Layer.merge( diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 52f7d7d43550..8db9dd50aac0 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -159,6 +159,27 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({ }); export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type; +export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + proof: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; + +export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + proof: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRegistrationRequest = + typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; + +export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ + endpoint: RelayManagedEndpoint, + endpointRuntime: RelayManagedEndpointRuntimeConfig, +}); +export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type; + export const RelayLinkProofRequest = Schema.Struct({ challenge: Schema.String, relayIssuer: Schema.String, @@ -186,6 +207,25 @@ const RelaySignedJwtRegisteredClaims = { exp: Schema.Int, } as const; +export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([ + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("register"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + }), + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("recover"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + }), +]); +export type RelayManagedEndpointRecoveryProofPayload = + typeof RelayManagedEndpointRecoveryProofPayload.Type; + export const RelayAgentActivityPublishProofPayload = Schema.Struct({ ...RelaySignedJwtRegisteredClaims, environmentId: EnvironmentId, @@ -1053,6 +1093,26 @@ export const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") export const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post( + "registerManagedEndpointRecovery", + "/v1/environments/:environmentId/tunnel/recovery", + { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRegistrationRequest, + success: RelayOkResponse, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"), + HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRequest, + success: RelayManagedEndpointRecoveryResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"), HttpApiEndpoint.post( "publishAgentActivity", "/v1/environments/:environmentId/threads/:threadId/agent-activity", diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index 9e848bedfb02..986bd982e622 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt"; export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; +export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; export class RelayJwtError extends Schema.TaggedErrorClass()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]),